Split per-app dataset into OS/data layers, add skiff rebuild

Each app now gets two ZFS datasets - a disposable OS layer
(zroot/skiff/appbase/<app>) cloned from base, and a persistent data
layer (zroot/skiff/appdata/<app>) holding releases/current/env/
meta.json/data.db. `skiff rebuild --app|--all` discards and reclones
the OS layer to pick up an OS/toolchain/config-template upgrade,
guarded by the same snapshot-and-rollback safety net as deploy and
config changes; the data layer is never touched.

Also fixes stale docs (builds already run via jexec inside the jail,
not on the host; --iface already auto-aliases the jail's IP, so the
manual ifconfig step and its troubleshooting entry are gone) found
while updating the same sections for this feature.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

5 files changed, 176 insertions, 2 deletions

diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go
index 30846ac..8d32613 100644
--- a/internal/deploy/deploy.go
+++ b/internal/deploy/deploy.go
@@ -113,7 +113,7 @@ func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string)
 // as the other if it goes wrong, so both are guarded the same way. label
 // is used only in the final error message (e.g. "deploy", "config change").
 func GuardedApply(ctx context.Context, out io.Writer, cfg layout.Config, app, tag, label string, apply func() error) error {
-	dataset := cfg.AppDataset(app)
+	dataset := cfg.AppDataDataset(app)
 
 	existing, err := zfsutil.Snapshots(ctx, dataset)
 	if err != nil {
diff --git a/internal/jailmgr/jailmgr.go b/internal/jailmgr/jailmgr.go
index 9ddf9b8..5da5663 100644
--- a/internal/jailmgr/jailmgr.go
+++ b/internal/jailmgr/jailmgr.go
@@ -68,6 +68,10 @@ func Start(ctx context.Context, out io.Writer, app string) error {
 	return sh.Run(ctx, out, "", "service", "jail", "start", app)
 }
 
+func Stop(ctx context.Context, out io.Writer, app string) error {
+	return sh.Run(ctx, out, "", "service", "jail", "stop", app)
+}
+
 // Restart restarts an already-running jail, or starts it if this is its
 // first deploy and it isn't up yet.
 func Restart(ctx context.Context, out io.Writer, app string) error {
diff --git a/internal/layout/layout.go b/internal/layout/layout.go
index 34edc32..5ac6c26 100644
--- a/internal/layout/layout.go
+++ b/internal/layout/layout.go
@@ -38,8 +38,19 @@ func Default() Config {
 	}
 }
 
+// AppDataset is the OS layer, cloned from BaseDataset and mounted at
+// JailRoot(app) - the jail's own root. Named "appbase" (not "apps") to
+// read clearly alongside AppDataDataset now that the two are siblings.
 func (c Config) AppDataset(app string) string {
-	return fmt.Sprintf("%s/apps/%s", c.ZFSPool, app)
+	return fmt.Sprintf("%s/appbase/%s", c.ZFSPool, app)
+}
+
+// AppDataDataset is the persistent data layer, mounted at AppDir(app):
+// releases/, current, env, meta.json, and the app's own data.db. It is a
+// separate dataset from AppDataset so `skiff rebuild` can destroy and
+// reclone the OS layer without touching it.
+func (c Config) AppDataDataset(app string) string {
+	return fmt.Sprintf("%s/appdata/%s", c.ZFSPool, app)
 }
 
 func (c Config) JailRoot(app string) string {
diff --git a/internal/rebuild/rebuild.go b/internal/rebuild/rebuild.go
new file mode 100644
index 0000000..f58c67f
--- /dev/null
+++ b/internal/rebuild/rebuild.go
@@ -0,0 +1,126 @@
+// Package rebuild implements `skiff rebuild`: discard an app's OS layer
+// (everything cloned from base) and reclone it fresh, leaving the data layer
+// (a separate dataset) untouched. Guarded the same way as
+// deploy.GuardedApply: the old OS dataset is kept until the new one is
+// confirmed healthy, and swapped back automatically on failure.
+package rebuild
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"skiff/internal/appmeta"
+	"skiff/internal/health"
+	"skiff/internal/jailmgr"
+	"skiff/internal/layout"
+	"skiff/internal/zfsutil"
+)
+
+func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, baseSnapshot, iface string) error {
+	osDataset := cfg.AppDataset(app)
+	osDatasetPrev := osDataset + "-prev"
+	jailRoot := cfg.JailRoot(app)
+	jailRootPrev := jailRoot + "-prev"
+	dataDataset := cfg.AppDataDataset(app)
+
+	// jail.conf.d/<app>.conf lives on the host, outside both per-app
+	// datasets, so swapping OS-layer datasets back on a failed rebuild
+	// doesn't revert it on its own - back it up so rollback can.
+	oldConf, err := os.ReadFile(cfg.JailConfPath(app))
+	if err != nil {
+		return fmt.Errorf("read current jail.conf: %w", err)
+	}
+
+	fmt.Fprintln(out, "==> stopping jail")
+	if err := jailmgr.Stop(ctx, out, app); err != nil {
+		return fmt.Errorf("stop jail: %w", err)
+	}
+	if err := zfsutil.Unmount(ctx, out, dataDataset); err != nil {
+		return fmt.Errorf("unmount data dataset: %w", err)
+	}
+
+	fmt.Fprintf(out, "==> setting aside current OS layer as %s\n", osDatasetPrev)
+	if err := zfsutil.Rename(ctx, out, osDataset, osDatasetPrev); err != nil {
+		return fmt.Errorf("rename OS dataset: %w", err)
+	}
+	// Rename doesn't move the mountpoint - relocate it explicitly, or the
+	// clone below silently mount-stacks on top of it instead of at jailRoot.
+	if err := zfsutil.SetMountpoint(ctx, out, osDatasetPrev, jailRootPrev); err != nil {
+		return fmt.Errorf("relocate old OS dataset mountpoint: %w", err)
+	}
+
+	fmt.Fprintf(out, "==> cloning fresh OS layer from %s@%s\n", cfg.BaseDataset, baseSnapshot)
+	if err := zfsutil.Clone(ctx, out, cfg.BaseDataset, baseSnapshot, osDataset, jailRoot); err != nil {
+		return fmt.Errorf("clone fresh OS dataset: %w", err)
+	}
+	if err := zfsutil.Mount(ctx, out, dataDataset); err != nil {
+		return fmt.Errorf("remount data dataset: %w", err)
+	}
+
+	if err := provisionAndStart(ctx, out, cfg, app, iface); err != nil {
+		fmt.Fprintf(out, "==> %v, rolling back to previous OS layer\n", err)
+		return rollback(ctx, out, cfg, app, osDataset, osDatasetPrev, jailRoot, dataDataset, oldConf, err)
+	}
+
+	fmt.Fprintf(out, "==> healthy, discarding previous OS layer %s\n", osDatasetPrev)
+	if err := zfsutil.Destroy(ctx, out, osDatasetPrev); err != nil {
+		return fmt.Errorf("rebuild succeeded but cleanup failed, destroy %s manually: %w", osDatasetPrev, err)
+	}
+	return nil
+}
+
+func provisionAndStart(ctx context.Context, out io.Writer, cfg layout.Config, app, iface string) error {
+	meta, err := appmeta.Read(cfg.MetaPath(app))
+	if err != nil {
+		return fmt.Errorf("read app metadata: %w", err)
+	}
+	if err := jailmgr.WriteConf(cfg, app, meta.IP, iface); err != nil {
+		return fmt.Errorf("write jail config: %w", err)
+	}
+	if err := jailmgr.EnableMail(cfg, app); err != nil {
+		return fmt.Errorf("configure mail: %w", err)
+	}
+	if err := jailmgr.Start(ctx, out, app); err != nil {
+		return fmt.Errorf("start jail: %w", err)
+	}
+	if err := health.WaitHealthy(ctx, meta); err != nil {
+		return fmt.Errorf("rebuilt jail failed health check: %w", err)
+	}
+	return nil
+}
+
+// rollback restores the previous OS layer, mirroring GuardedApply's failure
+// shape: the app ends up running on the dataset it was healthy on before
+// the attempt, and cause is always reported even if a rollback step fails.
+func rollback(ctx context.Context, out io.Writer, cfg layout.Config, app, osDataset, osDatasetPrev, jailRoot, dataDataset string, oldConf []byte, cause error) error {
+	fail := func(step string, err error) error {
+		return fmt.Errorf("rebuild failed AND rollback failed: %v / %s: %w", cause, step, err)
+	}
+	if err := jailmgr.Stop(ctx, out, app); err != nil {
+		return fail("stop jail", err)
+	}
+	if err := zfsutil.Unmount(ctx, out, dataDataset); err != nil {
+		return fail("unmount data dataset", err)
+	}
+	if err := zfsutil.Destroy(ctx, out, osDataset); err != nil {
+		return fail("destroy failed OS dataset", err)
+	}
+	if err := zfsutil.Rename(ctx, out, osDatasetPrev, osDataset); err != nil {
+		return fail("restore previous OS dataset", err)
+	}
+	if err := zfsutil.SetMountpoint(ctx, out, osDataset, jailRoot); err != nil {
+		return fail("restore OS dataset mountpoint", err)
+	}
+	if err := os.WriteFile(cfg.JailConfPath(app), oldConf, 0644); err != nil {
+		return fail("restore jail.conf", err)
+	}
+	if err := zfsutil.Mount(ctx, out, dataDataset); err != nil {
+		return fail("remount data dataset", err)
+	}
+	if err := jailmgr.Start(ctx, out, app); err != nil {
+		return fail("restart on previous OS layer", err)
+	}
+	return fmt.Errorf("rebuild failed and was rolled back to the previous OS layer: %w", cause)
+}
diff --git a/internal/zfsutil/zfsutil.go b/internal/zfsutil/zfsutil.go
index 918289c..5e68e30 100644
--- a/internal/zfsutil/zfsutil.go
+++ b/internal/zfsutil/zfsutil.go
@@ -41,6 +41,39 @@ func DestroySnapshot(ctx context.Context, out io.Writer, dataset, tag string) er
 	return sh.Run(ctx, out, "", "zfs", "destroy", snapRef(dataset, tag))
 }
 
+// Create makes a new, empty dataset mounted at mountpoint - unlike Clone,
+// it has no origin snapshot to inherit from.
+func Create(ctx context.Context, out io.Writer, dataset, mountpoint string) error {
+	return sh.Run(ctx, out, "", "zfs", "create", "-o", "mountpoint="+mountpoint, dataset)
+}
+
+// Rename renames a dataset. It deliberately does not touch the dataset's
+// mountpoint property, which zfs(8) leaves unchanged on a rename - callers
+// that care where the renamed dataset mounts must call SetMountpoint too.
+func Rename(ctx context.Context, out io.Writer, oldName, newName string) error {
+	return sh.Run(ctx, out, "", "zfs", "rename", oldName, newName)
+}
+
+func SetMountpoint(ctx context.Context, out io.Writer, dataset, mountpoint string) error {
+	return sh.Run(ctx, out, "", "zfs", "set", "mountpoint="+mountpoint, dataset)
+}
+
+func Mount(ctx context.Context, out io.Writer, dataset string) error {
+	return sh.Run(ctx, out, "", "zfs", "mount", dataset)
+}
+
+func Unmount(ctx context.Context, out io.Writer, dataset string) error {
+	return sh.Run(ctx, out, "", "zfs", "unmount", dataset)
+}
+
+// Destroy removes an entire dataset and any snapshots on it, as opposed to
+// DestroySnapshot's single dataset@tag. Recursive because callers use this
+// to discard a whole discarded OS-layer clone - an operator's own manual
+// snapshot on it shouldn't be able to block that.
+func Destroy(ctx context.Context, out io.Writer, dataset string) error {
+	return sh.Run(ctx, out, "", "zfs", "destroy", "-r", dataset)
+}
+
 // Snapshots lists snapshot names (without the dataset@ prefix), newest last.
 func Snapshots(ctx context.Context, dataset string) ([]string, error) {
 	raw, err := sh.Output(ctx, "", "zfs", "list", "-H", "-t", "snapshot", "-o", "name", "-s", "creation", "-r", dataset)