1 files changed, 101 insertions, 24 deletions

diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go
index 5430b7f..751c976 100644
--- a/internal/deploy/deploy.go
+++ b/internal/deploy/deploy.go
@@ -22,8 +22,6 @@ import (
 )
 
 func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string) error {
-	dataset := cfg.AppDataset(app)
-	snapshotTag := "pre-" + sha
 	releaseDir := cfg.ReleaseDir(app, sha)
 
 	fmt.Fprintf(out, "==> checking out %s\n", sha)
@@ -33,10 +31,37 @@ func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string)
 	if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "prune"); err != nil {
 		fmt.Fprintf(out, "==> warning: worktree prune failed: %v\n", err)
 	}
+	if _, err := os.Stat(releaseDir); err == nil {
+		// A retry of this exact sha whose previous attempt got far enough
+		// to cut over, then failed its health check: the dataset rollback
+		// reverts `current` but - by design - leaves this release dir (and
+		// its worktree registration) on disk, since it existed at
+		// snapshot time. Clear it so `add` below doesn't collide.
+		fmt.Fprintln(out, "==> removing leftover checkout from a previous attempt")
+		if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "remove", "--force", releaseDir); err != nil {
+			fmt.Fprintf(out, "==> warning: worktree remove failed (%v), removing directory directly\n", err)
+			if err := os.RemoveAll(releaseDir); err != nil {
+				return fmt.Errorf("remove leftover checkout: %w", err)
+			}
+		}
+	}
 	if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "add", releaseDir, sha); err != nil {
 		return fmt.Errorf("checkout: %w", err)
 	}
 
+	cutover := false
+	defer func() {
+		if cutover {
+			return
+		}
+		// Nothing ever went live under this checkout - leaving it behind
+		// blocks a retry of the same sha with "already exists" on the next
+		// `worktree add`.
+		if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "remove", "--force", releaseDir); err != nil {
+			fmt.Fprintf(out, "==> warning: failed to clean up worktree %s: %v\n", releaseDir, err)
+		}
+	}()
+
 	fmt.Fprintln(out, "==> building")
 	if err := sh.Run(ctx, out, releaseDir, "make", "-B"); err != nil {
 		return fmt.Errorf("build: %w", err)
@@ -45,27 +70,85 @@ func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string)
 		return fmt.Errorf("build: `make` did not produce ./app: %w", err)
 	}
 
-	fmt.Fprintln(out, "==> snapshotting current state")
-	if err := zfsutil.Snapshot(ctx, out, dataset, snapshotTag); err != nil {
-		return fmt.Errorf("snapshot: %w", err)
+	if err := GuardedApply(ctx, out, cfg, app, "pre-"+sha, "deploy", func() error {
+		hasRelease, err := release.Declared(ctx, releaseDir)
+		if err != nil {
+			return fmt.Errorf("check for release target: %w", err)
+		}
+		if hasRelease {
+			if err := release.Run(ctx, out, app, cfg.ReleaseDirInJail(sha)); err != nil {
+				return fmt.Errorf("release phase failed, deploy aborted before cutover: %w", err)
+			}
+		} else {
+			fmt.Fprintln(out, "==> no release target declared, skipping")
+		}
+
+		fmt.Fprintln(out, "==> cutting over")
+		if err := swapCurrent(cfg.CurrentRelease(app), cfg.ReleaseDirInJail(sha)); err != nil {
+			return fmt.Errorf("cutover: %w", err)
+		}
+		cutover = true
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	fmt.Fprintln(out, "==> healthy, updating routing")
+	meta, err := appmeta.Read(cfg.MetaPath(app))
+	if err != nil {
+		return fmt.Errorf("read app metadata: %w", err)
+	}
+	if err := nginx.EnsureRoute(ctx, out, cfg, app, meta.IP, meta.Port, meta.ServerName); err != nil {
+		return fmt.Errorf("nginx: %w", err)
 	}
 
-	hasRelease, err := release.Declared(ctx, releaseDir)
+	fmt.Fprintf(out, "==> deployed %s at %s\n", app, sha)
+	return nil
+}
+
+// GuardedApply snapshots app's dataset under tag, runs apply, restarts the
+// jail, and waits for it to report healthy - rolling the dataset back to
+// the snapshot (and restarting again) if it doesn't. Shared by code
+// deploys and config changes: either one is exactly as much of an outage
+// as the other if it goes wrong, so both are guarded the same way. label
+// is used only in the final error message (e.g. "deploy", "config change").
+func GuardedApply(ctx context.Context, out io.Writer, cfg layout.Config, app, tag, label string, apply func() error) error {
+	dataset := cfg.AppDataset(app)
+
+	existing, err := zfsutil.Snapshots(ctx, dataset)
 	if err != nil {
-		return fmt.Errorf("check for release target: %w", err)
+		return fmt.Errorf("list snapshots: %w", err)
 	}
-	if hasRelease {
-		if err := release.Run(ctx, out, app, cfg.ReleaseDirInJail(sha)); err != nil {
-			return fmt.Errorf("release phase failed, deploy aborted before cutover: %w", err)
+	for _, s := range existing {
+		if s != tag {
+			continue
+		}
+		// A previous attempt under this same tag (e.g. a retried deploy of
+		// the same sha, rolled back to this exact snapshot) left it
+		// behind - retaking it now just recaptures the state a rollback
+		// already reverted the dataset to.
+		fmt.Fprintf(out, "==> replacing stale snapshot %s from a previous attempt\n", tag)
+		if err := zfsutil.DestroySnapshot(ctx, out, dataset, tag); err != nil {
+			return fmt.Errorf("clean up stale snapshot: %w", err)
 		}
-	} else {
-		fmt.Fprintln(out, "==> no release target declared, skipping")
+		break
 	}
 
-	fmt.Fprintln(out, "==> cutting over")
-	if err := swapCurrent(cfg.CurrentRelease(app), cfg.ReleaseDirInJail(sha)); err != nil {
-		return fmt.Errorf("cutover: %w", err)
+	fmt.Fprintln(out, "==> snapshotting current state")
+	if err := zfsutil.Snapshot(ctx, out, dataset, tag); err != nil {
+		return fmt.Errorf("snapshot: %w", err)
 	}
+
+	if err := apply(); err != nil {
+		// Nothing has been restarted yet, so this snapshot was never needed
+		// as a rollback target - left behind, it collides with the same
+		// tag on the next retry.
+		if destroyErr := zfsutil.DestroySnapshot(ctx, out, dataset, tag); destroyErr != nil {
+			fmt.Fprintf(out, "==> warning: failed to clean up snapshot %s: %v\n", tag, destroyErr)
+		}
+		return err
+	}
+
 	if err := jailmgr.Restart(ctx, out, app); err != nil {
 		return fmt.Errorf("restart: %w", err)
 	}
@@ -76,22 +159,16 @@ func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string)
 	}
 
 	if err := health.WaitHealthy(ctx, meta); err != nil {
-		fmt.Fprintf(out, "==> health check failed (%v), rolling back to pre-%s\n", err, sha)
-		if rbErr := zfsutil.Rollback(ctx, out, dataset, snapshotTag); rbErr != nil {
+		fmt.Fprintf(out, "==> health check failed (%v), rolling back to %s\n", err, tag)
+		if rbErr := zfsutil.Rollback(ctx, out, dataset, tag); rbErr != nil {
 			return fmt.Errorf("health check failed AND rollback failed: %v / %w", err, rbErr)
 		}
 		if rbErr := jailmgr.Restart(ctx, out, app); rbErr != nil {
 			return fmt.Errorf("health check failed, rolled back, but restart failed: %v / %w", err, rbErr)
 		}
-		return fmt.Errorf("deploy failed health check and was rolled back: %w", err)
-	}
-
-	fmt.Fprintln(out, "==> healthy, updating routing")
-	if err := nginx.EnsureRoute(ctx, out, cfg, app, meta.IP, meta.Port, meta.ServerName); err != nil {
-		return fmt.Errorf("nginx: %w", err)
+		return fmt.Errorf("%s failed health check and was rolled back: %w", label, err)
 	}
 
-	fmt.Fprintf(out, "==> deployed %s at %s\n", app, sha)
 	return nil
 }