From d1e1c226d9a63432c652d373c039a3ae4430f3a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 00:03:04 -0500 Subject: Guard deploys and config changes against leftover state on retry - Extract deploy.Run's snapshot -> apply -> restart -> health-check -> rollback-on-failure sequence into deploy.GuardedApply, shared with cmd/skiff's applyConfigChange, which duplicated it verbatim. - Clean up the worktree and snapshot a deploy creates when it fails before cutover (e.g. a failing release-phase migration), so retrying the same sha no longer needs manual cleanup. - Make checkout and snapshotting idempotent for a same-sha retry after a cutover that later got rolled back on a failed health check, where the release dir and snapshot are deliberately left behind by design. - Document the actual cause of a jail not surviving a host reboot: FreeBSD ships no /etc/jail.conf, so boot-time `service jail start` finds nothing to start regardless of jail.conf.d - only name-targeted starts read jail.conf.d directly, which is why manual starts worked. --- cmd/skiff/main.go | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) (limited to 'cmd/skiff') diff --git a/cmd/skiff/main.go b/cmd/skiff/main.go index 2fe5907..1c48681 100644 --- a/cmd/skiff/main.go +++ b/cmd/skiff/main.go @@ -14,7 +14,6 @@ import ( "skiff/internal/appmeta" "skiff/internal/deploy" "skiff/internal/envfile" - "skiff/internal/health" "skiff/internal/jailmgr" "skiff/internal/layout" "skiff/internal/sh" @@ -266,9 +265,10 @@ func cmdConfigList(args []string) error { } // applyConfigChange loads app's current env file, applies mutate to it, and -// carries the same snapshot -> restart -> health-check -> rollback-on- -// failure sequence deploy.Run uses for code deploys - a bad config change -// is exactly as much of an outage as a bad deploy, guarded the same way. +// saves it under deploy.GuardedApply - the same snapshot -> restart -> +// health-check -> rollback-on-failure sequence deploy.Run uses for code +// deploys, since a bad config change is exactly as much of an outage as a +// bad deploy. func applyConfigChange(app string, mutate func(map[string]string) error) error { cfg := layout.Default() if _, err := os.Stat(cfg.MetaPath(app)); err != nil { @@ -285,36 +285,15 @@ func applyConfigChange(app string, mutate func(map[string]string) error) error { ctx := context.Background() out := os.Stdout - dataset := cfg.AppDataset(app) tag := fmt.Sprintf("precfg-%d", time.Now().Unix()) - fmt.Fprintln(out, "==> snapshotting current state") - if err := zfsutil.Snapshot(ctx, out, dataset, tag); err != nil { - return fmt.Errorf("snapshot: %w", err) - } - - if err := envfile.Save(cfg.EnvPath(app), kv); err != nil { - return fmt.Errorf("save config: %w", err) - } - - if err := jailmgr.Restart(ctx, out, app); err != nil { - return fmt.Errorf("restart: %w", err) - } - - meta, err := appmeta.Read(cfg.MetaPath(app)) - if err != nil { - return fmt.Errorf("read app metadata: %w", err) - } - - if err := health.WaitHealthy(ctx, meta); err != nil { - fmt.Fprintf(out, "==> health check failed (%v), rolling back to %s\n", err, tag) - if rbErr := zfsutil.Rollback(ctx, out, dataset, tag); rbErr != nil { - return fmt.Errorf("health check failed AND rollback failed: %v / %w", err, rbErr) - } - if rbErr := jailmgr.Restart(ctx, out, app); rbErr != nil { - return fmt.Errorf("health check failed, rolled back, but restart failed: %v / %w", err, rbErr) + if err := deploy.GuardedApply(ctx, out, cfg, app, tag, "config change", func() error { + if err := envfile.Save(cfg.EnvPath(app), kv); err != nil { + return fmt.Errorf("save config: %w", err) } - return fmt.Errorf("config change failed health check and was rolled back: %w", err) + return nil + }); err != nil { + return err } fmt.Fprintln(out, "==> config updated") -- cgit v1.3