1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
// Package rebuild implements `skiff rebuild`: discard an app's OS layer
// (everything cloned from base) and reclone it fresh, leaving the data layer
// (a separate dataset) untouched. Guarded the same way as
// deploy.GuardedApply: the old OS dataset is kept until the new one is
// confirmed healthy, and swapped back automatically on failure.
package rebuild
import (
"context"
"fmt"
"io"
"os"
"skiff/internal/appmeta"
"skiff/internal/health"
"skiff/internal/jailmgr"
"skiff/internal/layout"
"skiff/internal/zfsutil"
)
func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, baseSnapshot, iface string) error {
osDataset := cfg.AppDataset(app)
osDatasetPrev := osDataset + "-prev"
jailRoot := cfg.JailRoot(app)
jailRootPrev := jailRoot + "-prev"
dataDataset := cfg.AppDataDataset(app)
// jail.conf.d/<app>.conf lives on the host, outside both per-app
// datasets, so swapping OS-layer datasets back on a failed rebuild
// doesn't revert it on its own - back it up so rollback can.
oldConf, err := os.ReadFile(cfg.JailConfPath(app))
if err != nil {
return fmt.Errorf("read current jail.conf: %w", err)
}
fmt.Fprintln(out, "==> stopping jail")
if err := jailmgr.Stop(ctx, out, app); err != nil {
return fmt.Errorf("stop jail: %w", err)
}
if err := zfsutil.Unmount(ctx, out, dataDataset); err != nil {
return fmt.Errorf("unmount data dataset: %w", err)
}
fmt.Fprintf(out, "==> setting aside current OS layer as %s\n", osDatasetPrev)
if err := zfsutil.Rename(ctx, out, osDataset, osDatasetPrev); err != nil {
return fmt.Errorf("rename OS dataset: %w", err)
}
// Rename doesn't move the mountpoint - relocate it explicitly, or the
// clone below silently mount-stacks on top of it instead of at jailRoot.
if err := zfsutil.SetMountpoint(ctx, out, osDatasetPrev, jailRootPrev); err != nil {
return fmt.Errorf("relocate old OS dataset mountpoint: %w", err)
}
fmt.Fprintf(out, "==> cloning fresh OS layer from %s@%s\n", cfg.BaseDataset, baseSnapshot)
if err := zfsutil.Clone(ctx, out, cfg.BaseDataset, baseSnapshot, osDataset, jailRoot); err != nil {
return fmt.Errorf("clone fresh OS dataset: %w", err)
}
if err := zfsutil.Mount(ctx, out, dataDataset); err != nil {
return fmt.Errorf("remount data dataset: %w", err)
}
if err := provisionAndStart(ctx, out, cfg, app, iface); err != nil {
fmt.Fprintf(out, "==> %v, rolling back to previous OS layer\n", err)
return rollback(ctx, out, cfg, app, osDataset, osDatasetPrev, jailRoot, dataDataset, oldConf, err)
}
fmt.Fprintf(out, "==> healthy, discarding previous OS layer %s\n", osDatasetPrev)
if err := zfsutil.Destroy(ctx, out, osDatasetPrev); err != nil {
return fmt.Errorf("rebuild succeeded but cleanup failed, destroy %s manually: %w", osDatasetPrev, err)
}
return nil
}
func provisionAndStart(ctx context.Context, out io.Writer, cfg layout.Config, app, iface string) error {
meta, err := appmeta.Read(cfg.MetaPath(app))
if err != nil {
return fmt.Errorf("read app metadata: %w", err)
}
if err := jailmgr.WriteConf(cfg, app, meta.IP, iface); err != nil {
return fmt.Errorf("write jail config: %w", err)
}
if err := jailmgr.EnableMail(cfg, app); err != nil {
return fmt.Errorf("configure mail: %w", err)
}
if err := jailmgr.Start(ctx, out, app); err != nil {
return fmt.Errorf("start jail: %w", err)
}
if err := health.WaitHealthy(ctx, meta); err != nil {
return fmt.Errorf("rebuilt jail failed health check: %w", err)
}
return nil
}
// rollback restores the previous OS layer, mirroring GuardedApply's failure
// shape: the app ends up running on the dataset it was healthy on before
// the attempt, and cause is always reported even if a rollback step fails.
func rollback(ctx context.Context, out io.Writer, cfg layout.Config, app, osDataset, osDatasetPrev, jailRoot, dataDataset string, oldConf []byte, cause error) error {
fail := func(step string, err error) error {
return fmt.Errorf("rebuild failed AND rollback failed: %v / %s: %w", cause, step, err)
}
if err := jailmgr.Stop(ctx, out, app); err != nil {
return fail("stop jail", err)
}
if err := zfsutil.Unmount(ctx, out, dataDataset); err != nil {
return fail("unmount data dataset", err)
}
if err := zfsutil.Destroy(ctx, out, osDataset); err != nil {
return fail("destroy failed OS dataset", err)
}
if err := zfsutil.Rename(ctx, out, osDatasetPrev, osDataset); err != nil {
return fail("restore previous OS dataset", err)
}
if err := zfsutil.SetMountpoint(ctx, out, osDataset, jailRoot); err != nil {
return fail("restore OS dataset mountpoint", err)
}
if err := os.WriteFile(cfg.JailConfPath(app), oldConf, 0644); err != nil {
return fail("restore jail.conf", err)
}
if err := zfsutil.Mount(ctx, out, dataDataset); err != nil {
return fail("remount data dataset", err)
}
if err := jailmgr.Start(ctx, out, app); err != nil {
return fail("restart on previous OS layer", err)
}
return fmt.Errorf("rebuild failed and was rolled back to the previous OS layer: %w", cause)
}
|
