Guard deploys and config changes against leftover state on retry

- Extract deploy.Run's snapshot -> apply -> restart -> health-check ->
  rollback-on-failure sequence into deploy.GuardedApply, shared with
  cmd/skiff's applyConfigChange, which duplicated it verbatim.
- Clean up the worktree and snapshot a deploy creates when it fails
  before cutover (e.g. a failing release-phase migration), so retrying
  the same sha no longer needs manual cleanup.
- Make checkout and snapshotting idempotent for a same-sha retry after
  a cutover that later got rolled back on a failed health check, where
  the release dir and snapshot are deliberately left behind by design.
- Document the actual cause of a jail not surviving a host reboot:
  FreeBSD ships no /etc/jail.conf, so boot-time `service jail start`
  finds nothing to start regardless of jail.conf.d - only name-targeted
  starts read jail.conf.d directly, which is why manual starts worked.

1 files changed, 10 insertions, 31 deletions

diff --git a/cmd/skiff/main.go b/cmd/skiff/main.go
index 2fe5907..1c48681 100644
--- a/cmd/skiff/main.go
+++ b/cmd/skiff/main.go
@@ -14,7 +14,6 @@ import (
 	"skiff/internal/appmeta"
 	"skiff/internal/deploy"
 	"skiff/internal/envfile"
-	"skiff/internal/health"
 	"skiff/internal/jailmgr"
 	"skiff/internal/layout"
 	"skiff/internal/sh"
@@ -266,9 +265,10 @@ func cmdConfigList(args []string) error {
 }
 
 // applyConfigChange loads app's current env file, applies mutate to it, and
-// carries the same snapshot -> restart -> health-check -> rollback-on-
-// failure sequence deploy.Run uses for code deploys - a bad config change
-// is exactly as much of an outage as a bad deploy, guarded the same way.
+// saves it under deploy.GuardedApply - the same snapshot -> restart ->
+// health-check -> rollback-on-failure sequence deploy.Run uses for code
+// deploys, since a bad config change is exactly as much of an outage as a
+// bad deploy.
 func applyConfigChange(app string, mutate func(map[string]string) error) error {
 	cfg := layout.Default()
 	if _, err := os.Stat(cfg.MetaPath(app)); err != nil {
@@ -285,36 +285,15 @@ func applyConfigChange(app string, mutate func(map[string]string) error) error {
 
 	ctx := context.Background()
 	out := os.Stdout
-	dataset := cfg.AppDataset(app)
 	tag := fmt.Sprintf("precfg-%d", time.Now().Unix())
 
-	fmt.Fprintln(out, "==> snapshotting current state")
-	if err := zfsutil.Snapshot(ctx, out, dataset, tag); err != nil {
-		return fmt.Errorf("snapshot: %w", err)
-	}
-
-	if err := envfile.Save(cfg.EnvPath(app), kv); err != nil {
-		return fmt.Errorf("save config: %w", err)
-	}
-
-	if err := jailmgr.Restart(ctx, out, app); err != nil {
-		return fmt.Errorf("restart: %w", err)
-	}
-
-	meta, err := appmeta.Read(cfg.MetaPath(app))
-	if err != nil {
-		return fmt.Errorf("read app metadata: %w", err)
-	}
-
-	if err := health.WaitHealthy(ctx, meta); err != nil {
-		fmt.Fprintf(out, "==> health check failed (%v), rolling back to %s\n", err, tag)
-		if rbErr := zfsutil.Rollback(ctx, out, dataset, tag); rbErr != nil {
-			return fmt.Errorf("health check failed AND rollback failed: %v / %w", err, rbErr)
-		}
-		if rbErr := jailmgr.Restart(ctx, out, app); rbErr != nil {
-			return fmt.Errorf("health check failed, rolled back, but restart failed: %v / %w", err, rbErr)
+	if err := deploy.GuardedApply(ctx, out, cfg, app, tag, "config change", func() error {
+		if err := envfile.Save(cfg.EnvPath(app), kv); err != nil {
+			return fmt.Errorf("save config: %w", err)
 		}
-		return fmt.Errorf("config change failed health check and was rolled back: %w", err)
+		return nil
+	}); err != nil {
+		return err
 	}
 
 	fmt.Fprintln(out, "==> config updated")