Split per-app dataset into OS/data layers, add skiff rebuild

Each app now gets two ZFS datasets - a disposable OS layer
(zroot/skiff/appbase/<app>) cloned from base, and a persistent data
layer (zroot/skiff/appdata/<app>) holding releases/current/env/
meta.json/data.db. `skiff rebuild --app|--all` discards and reclones
the OS layer to pick up an OS/toolchain/config-template upgrade,
guarded by the same snapshot-and-rollback safety net as deploy and
config changes; the data layer is never touched.

Also fixes stale docs (builds already run via jexec inside the jail,
not on the host; --iface already auto-aliases the jail's IP, so the
manual ifconfig step and its troubleshooting entry are gone) found
while updating the same sections for this feature.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

1 files changed, 69 insertions, 2 deletions

diff --git a/cmd/skiff/main.go b/cmd/skiff/main.go
index 8ca1406..3286da7 100644
--- a/cmd/skiff/main.go
+++ b/cmd/skiff/main.go
@@ -7,6 +7,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -17,6 +18,7 @@ import (
 	"skiff/internal/jailmgr"
 	"skiff/internal/layout"
 	"skiff/internal/lock"
+	"skiff/internal/rebuild"
 	"skiff/internal/sh"
 	"skiff/internal/tmpl"
 	"skiff/internal/zfsutil"
@@ -24,7 +26,7 @@ import (
 
 func main() {
 	if len(os.Args) < 2 {
-		fmt.Fprintln(os.Stderr, "usage: skiff <deploy|new-app|rollback|config> [flags]")
+		fmt.Fprintln(os.Stderr, "usage: skiff <deploy|new-app|rollback|rebuild|config> [flags]")
 		os.Exit(1)
 	}
 
@@ -36,6 +38,8 @@ func main() {
 		err = cmdNewApp(os.Args[2:])
 	case "rollback":
 		err = cmdRollback(os.Args[2:])
+	case "rebuild":
+		err = cmdRebuild(os.Args[2:])
 	case "config":
 		err = cmdConfig(os.Args[2:])
 	default:
@@ -109,6 +113,11 @@ func cmdNewApp(args []string) error {
 		return fmt.Errorf("clone base dataset: %w", err)
 	}
 
+	fmt.Fprintf(out, "==> creating data dataset %s\n", cfg.AppDataDataset(*name))
+	if err := zfsutil.Create(ctx, out, cfg.AppDataDataset(*name), cfg.AppDir(*name)); err != nil {
+		return fmt.Errorf("create data dataset: %w", err)
+	}
+
 	if err := os.MkdirAll(cfg.ReleasesDir(*name), 0755); err != nil {
 		return fmt.Errorf("create app dir: %w", err)
 	}
@@ -165,7 +174,7 @@ func cmdRollback(args []string) error {
 	}
 	ctx := context.Background()
 	cfg := layout.Default()
-	dataset := cfg.AppDataset(*app)
+	dataset := cfg.AppDataDataset(*app)
 
 	if *list {
 		snaps, err := zfsutil.Snapshots(ctx, dataset)
@@ -191,6 +200,64 @@ func cmdRollback(args []string) error {
 	return jailmgr.Restart(ctx, os.Stdout, *app)
 }
 
+func cmdRebuild(args []string) error {
+	fs := flag.NewFlagSet("rebuild", flag.ExitOnError)
+	app := fs.String("app", "", "app name")
+	all := fs.Bool("all", false, "rebuild every app")
+	baseSnapshot := fs.String("base-snapshot", "ready", "snapshot on the base dataset to clone from")
+	iface := fs.String("iface", layout.DefaultJailIface, "host interface the jail's IP is aliased onto")
+	fs.Parse(args)
+
+	if (*app == "") != *all {
+		return fmt.Errorf("exactly one of --app or --all is required")
+	}
+
+	cfg := layout.Default()
+	if *app != "" {
+		return rebuildOne(cfg, *app, *baseSnapshot, *iface)
+	}
+
+	apps, err := listApps(cfg)
+	if err != nil {
+		return fmt.Errorf("list apps: %w", err)
+	}
+	var done []string
+	for _, a := range apps {
+		fmt.Printf("==> rebuilding %s\n", a)
+		if err := rebuildOne(cfg, a, *baseSnapshot, *iface); err != nil {
+			fmt.Printf("==> rebuilt %d/%d apps before %s failed\n", len(done), len(apps), a)
+			return fmt.Errorf("rebuild %s: %w", a, err)
+		}
+		done = append(done, a)
+	}
+	fmt.Printf("==> rebuilt %d/%d apps\n", len(done), len(apps))
+	return nil
+}
+
+func rebuildOne(cfg layout.Config, app, baseSnapshot, iface string) error {
+	unlock, err := lock.Acquire(cfg.LockPath(app))
+	if err != nil {
+		return fmt.Errorf("acquire app lock: %w", err)
+	}
+	defer unlock.Release()
+	return rebuild.Run(context.Background(), os.Stdout, cfg, app, baseSnapshot, iface)
+}
+
+// listApps has no other caller today - every other command already knows
+// its app name via --app.
+func listApps(cfg layout.Config) ([]string, error) {
+	matches, err := filepath.Glob(filepath.Join(cfg.JailConfDir, "*.conf"))
+	if err != nil {
+		return nil, err
+	}
+	apps := make([]string, len(matches))
+	for i, m := range matches {
+		apps[i] = strings.TrimSuffix(filepath.Base(m), ".conf")
+	}
+	sort.Strings(apps)
+	return apps, nil
+}
+
 func cmdConfig(args []string) error {
 	if len(args) < 1 {
 		return fmt.Errorf("usage: skiff config <set|unset|list> --app <name> [args...]")