1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
// Package deploy orchestrates one push: checkout, build, snapshot, release
// phase, cutover, health check, and - on a failed health check - a full
// dataset rollback. It is the entire "control plane": one function, called
// synchronously from the repo's post-receive hook.
package deploy
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"skiff/internal/appmeta"
"skiff/internal/health"
"skiff/internal/jailmgr"
"skiff/internal/layout"
"skiff/internal/nginx"
"skiff/internal/release"
"skiff/internal/sh"
"skiff/internal/zfsutil"
)
func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string) error {
releaseDir := cfg.ReleaseDir(app, sha)
fmt.Fprintf(out, "==> checking out %s\n", sha)
// A previous rollback can revert the app's dataset (and so delete a
// release dir) without the bare repo - on a separate dataset - ever
// knowing; prune first or `worktree add` fails on the stale entry.
if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "prune"); err != nil {
fmt.Fprintf(out, "==> warning: worktree prune failed: %v\n", err)
}
if _, err := os.Stat(releaseDir); err == nil {
// A retry of this exact sha whose previous attempt got far enough
// to cut over, then failed its health check: the dataset rollback
// reverts `current` but - by design - leaves this release dir (and
// its worktree registration) on disk, since it existed at
// snapshot time. Clear it so `add` below doesn't collide.
fmt.Fprintln(out, "==> removing leftover checkout from a previous attempt")
if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "remove", "--force", releaseDir); err != nil {
fmt.Fprintf(out, "==> warning: worktree remove failed (%v), removing directory directly\n", err)
if err := os.RemoveAll(releaseDir); err != nil {
return fmt.Errorf("remove leftover checkout: %w", err)
}
}
}
if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "add", releaseDir, sha); err != nil {
return fmt.Errorf("checkout: %w", err)
}
cutover := false
defer func() {
if cutover {
return
}
// Nothing ever went live under this checkout - leaving it behind
// blocks a retry of the same sha with "already exists" on the next
// `worktree add`.
if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "remove", "--force", releaseDir); err != nil {
fmt.Fprintf(out, "==> warning: failed to clean up worktree %s: %v\n", releaseDir, err)
}
}()
fmt.Fprintln(out, "==> building")
if err := sh.Run(ctx, out, "", "jexec", app, "make", "-B", "-C", cfg.ReleaseDirInJail(sha)); err != nil {
return fmt.Errorf("build: %w", err)
}
if _, err := os.Stat(filepath.Join(releaseDir, "app")); err != nil {
return fmt.Errorf("build: `make` did not produce ./app: %w", err)
}
if err := GuardedApply(ctx, out, cfg, app, "pre-"+sha, "deploy", func() error {
hasRelease, err := release.Declared(ctx, releaseDir)
if err != nil {
return fmt.Errorf("check for release target: %w", err)
}
if hasRelease {
if err := release.Run(ctx, out, app, cfg.ReleaseDirInJail(sha)); err != nil {
return fmt.Errorf("release phase failed, deploy aborted before cutover: %w", err)
}
} else {
fmt.Fprintln(out, "==> no release target declared, skipping")
}
fmt.Fprintln(out, "==> cutting over")
if err := swapCurrent(cfg.CurrentRelease(app), cfg.ReleaseDirInJail(sha)); err != nil {
return fmt.Errorf("cutover: %w", err)
}
cutover = true
return nil
}); err != nil {
return err
}
fmt.Fprintln(out, "==> healthy, updating routing")
meta, err := appmeta.Read(cfg.MetaPath(app))
if err != nil {
return fmt.Errorf("read app metadata: %w", err)
}
if err := nginx.EnsureRoute(ctx, out, cfg, app, meta.IP, meta.Port, meta.ServerName); err != nil {
return fmt.Errorf("nginx: %w", err)
}
fmt.Fprintf(out, "==> deployed %s at %s\n", app, sha)
return nil
}
// GuardedApply snapshots app's dataset under tag, runs apply, restarts the
// jail, and waits for it to report healthy - rolling the dataset back to
// the snapshot (and restarting again) if it doesn't. Shared by code
// deploys and config changes: either one is exactly as much of an outage
// as the other if it goes wrong, so both are guarded the same way. label
// is used only in the final error message (e.g. "deploy", "config change").
func GuardedApply(ctx context.Context, out io.Writer, cfg layout.Config, app, tag, label string, apply func() error) error {
dataset := cfg.AppDataset(app)
existing, err := zfsutil.Snapshots(ctx, dataset)
if err != nil {
return fmt.Errorf("list snapshots: %w", err)
}
for _, s := range existing {
if s != tag {
continue
}
// A previous attempt under this same tag (e.g. a retried deploy of
// the same sha, rolled back to this exact snapshot) left it
// behind - retaking it now just recaptures the state a rollback
// already reverted the dataset to.
fmt.Fprintf(out, "==> replacing stale snapshot %s from a previous attempt\n", tag)
if err := zfsutil.DestroySnapshot(ctx, out, dataset, tag); err != nil {
return fmt.Errorf("clean up stale snapshot: %w", err)
}
break
}
fmt.Fprintln(out, "==> snapshotting current state")
if err := zfsutil.Snapshot(ctx, out, dataset, tag); err != nil {
return fmt.Errorf("snapshot: %w", err)
}
if err := apply(); err != nil {
// Nothing has been restarted yet, so this snapshot was never needed
// as a rollback target - left behind, it collides with the same
// tag on the next retry.
if destroyErr := zfsutil.DestroySnapshot(ctx, out, dataset, tag); destroyErr != nil {
fmt.Fprintf(out, "==> warning: failed to clean up snapshot %s: %v\n", tag, destroyErr)
}
return err
}
if err := jailmgr.Restart(ctx, out, app); err != nil {
return fmt.Errorf("restart: %w", err)
}
meta, err := appmeta.Read(cfg.MetaPath(app))
if err != nil {
return fmt.Errorf("read app metadata: %w", err)
}
if err := health.WaitHealthy(ctx, meta); err != nil {
fmt.Fprintf(out, "==> health check failed (%v), rolling back to %s\n", err, tag)
if rbErr := zfsutil.Rollback(ctx, out, dataset, tag); rbErr != nil {
return fmt.Errorf("health check failed AND rollback failed: %v / %w", err, rbErr)
}
if rbErr := jailmgr.Restart(ctx, out, app); rbErr != nil {
return fmt.Errorf("health check failed, rolled back, but restart failed: %v / %w", err, rbErr)
}
return fmt.Errorf("%s failed health check and was rolled back: %w", label, err)
}
return nil
}
// swapCurrent atomically repoints the `current` symlink at newTarget by
// building the replacement next to it and renaming over it - never leaves
// the symlink briefly missing.
func swapCurrent(currentPath, newTarget string) error {
tmp := currentPath + ".next"
os.Remove(tmp)
if err := os.Symlink(newTarget, tmp); err != nil {
return err
}
return os.Rename(tmp, currentPath)
}
|
