Skip to content

Commit f16580b

Browse files
authored
feat: support process cgroup isolation without systemd (#11)
2 parents a43066c + 7489b88 commit f16580b

25 files changed

Lines changed: 530 additions & 111 deletions

File tree

addons/containerd/go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
214214
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
215215
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
216216
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
217+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
217218
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
218219
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
219220
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

addons/docker/go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
109109
google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 h1:APHvLLYBhtZvsbnpkfknDZ7NyH4z5+ub/I0u8L3Oz6g=
110110
google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1/go.mod h1:xUjFWUnWDpZ/C0Gu0qloASKFb6f8/QXiiXhSPFsD668=
111111
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
112+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
112113
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
113114
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
114115
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=

addons/gcloud/addon.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -271,8 +271,8 @@ func copyADC(ctx context.Context) error {
271271
if err != nil {
272272
return err
273273
}
274-
defer os.Remove(tf.Name())
275-
defer tf.Close()
274+
defer os.Remove(tf.Name()) //nolint:errcheck
275+
defer tf.Close() //nolint:errcheck
276276
if _, err := tf.Write(adcContents); err != nil {
277277
return err
278278
}

addons/gcs/go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc
6464
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
6565
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
6666
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
67+
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
6768
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
6869
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
6970
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

addons/gocache/gcs/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,13 @@ golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
125125
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
126126
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
127127
google.golang.org/api v0.255.0 h1:OaF+IbRwOottVCYV2wZan7KUq7UeNUQn1BcPc4K7lE4=
128+
google.golang.org/api v0.255.0/go.mod h1:d1/EtvCLdtiWEV4rAEHDHGh2bCnqsWhw+M8y2ECN4a8=
128129
google.golang.org/genproto v0.0.0-20250826171959-ef028d996bc1 h1:Nm5SEGIguOIBDXs5rhfz2aKwEVWlgwC58UcmEnLDc8Y=
129130
google.golang.org/genproto v0.0.0-20250826171959-ef028d996bc1/go.mod h1:Jz9LrroM7Mcm+a0QrLh4UpZ1B/WhjIbqwEcUf4y08nQ=
130131
google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 h1:APHvLLYBhtZvsbnpkfknDZ7NyH4z5+ub/I0u8L3Oz6g=
131132
google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1/go.mod h1:xUjFWUnWDpZ/C0Gu0qloASKFb6f8/QXiiXhSPFsD668=
132133
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
134+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
133135
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
134136
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
135137
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=

addons/gocache/layered-backend.go

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -167,22 +167,19 @@ func (l *layeredStorageBackend) WriteOutput(a ActionEntry, body io.Reader) (stri
167167
var localPath string
168168
var err1, err2 error
169169
var wg sync.WaitGroup
170-
wg.Add(2)
171-
go func() {
172-
defer wg.Done()
170+
wg.Go(func() {
173171
defer bodyCopyW.Close() // nolint:errcheck // else pipe won't know we're done
174172
defer io.Copy(io.Discard, body) // nolint:errcheck // drain the body to avoid deadlock
175173
if localPath, err1 = l.localW.WriteOutput(a, body); err1 != nil {
176174
err1 = fmt.Errorf("failed to write output to local storage: %w", err1)
177175
}
178-
}()
179-
go func() {
180-
defer wg.Done()
176+
})
177+
wg.Go(func() {
181178
defer io.Copy(io.Discard, bodyCopyR) // nolint:errcheck // drain the body to avoid deadlock
182179
if _, err2 = l.remoteW.WriteOutput(a, bodyCopyR); err2 != nil {
183180
err2 = fmt.Errorf("failed to write output to remote storage: %w", err2)
184181
}
185-
}()
182+
})
186183
wg.Wait()
187184
return localPath, errors.Join(err1, err2)
188185
}

addons/pm/api/child.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ type ExecStatus struct {
3737
State ExecState `json:"state"`
3838
StartErr string `json:"startErr"`
3939
Pid int `json:"pid,omitzero"`
40+
Group string `json:"group,omitzero"`
4041
ExitCode int `json:"exitCode"`
4142
}
4243

addons/pm/server/child.go

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,23 @@ import (
2222
)
2323

2424
type child struct {
25-
def api.Child
26-
status atomic.Pointer[api.ChildStatus]
27-
cmds chan childCmd
28-
wg sync.WaitGroup
25+
def api.Child
26+
status atomic.Pointer[api.ChildStatus]
27+
cmds chan childCmd
28+
wg sync.WaitGroup
29+
isolator isolator
2930

3031
restartDelay time.Duration
3132
killDelay time.Duration
3233
healthCheckInitialInterval time.Duration
3334
healthCheckInterval time.Duration
3435
}
3536

36-
func newChild(def api.Child) *child {
37+
func newChild(def api.Child, isolator isolator) *child {
3738
c := &child{
38-
def: def,
39-
cmds: make(chan childCmd), // important that this be un-buffered
39+
def: def,
40+
cmds: make(chan childCmd), // important that this be un-buffered
41+
isolator: isolator,
4042

4143
// tests may override these
4244
restartDelay: time.Second, // TODO: scale
@@ -142,6 +144,9 @@ MANAGER:
142144
}
143145
curProc = nil
144146
s := curStatus()
147+
// make sure any children that tried to fork off get caught and killed via
148+
// the cgroup, unless they managed to escape into a new cgroup
149+
c.cleanup(s)
145150
s.State = api.ExecEnded
146151
var ee *exec.ExitError
147152
if errors.As(err, &ee) {
@@ -153,6 +158,9 @@ MANAGER:
153158
s.Pid = 0
154159
switch status.State {
155160
case api.ChildStopping:
161+
// re-check all the isolation groups to make sure all processes are
162+
// killed and cgroups removed
163+
c.cleanupAll(&status)
156164
// stop completed
157165
status.State = api.ChildStopped
158166
// reset the starting process to the beginning
@@ -315,21 +323,20 @@ func (c *child) start(
315323
return nil, api.ExecStatus{State: api.ExecNotStarted, StartErr: err.Error()}, errorState
316324
}
317325
log.Printf("started %s as pid %d", name, cmd.Process.Pid)
318-
c.wg.Add(1)
319-
go func() {
320-
defer c.wg.Done()
326+
c.wg.Go(func() {
321327
err := cmd.Wait()
322328
exited <- err
323-
}()
324-
if err := isolateProcess(context.TODO(), name, cmd.Process); err != nil {
329+
})
330+
eStat := api.ExecStatus{
331+
State: api.ExecRunning,
332+
Pid: cmd.Process.Pid,
333+
}
334+
if isolationGroup, err := c.isolator.isolateProcess(context.TODO(), name, cmd.Process); err != nil {
325335
log.Printf("ERROR: failed to isolate process %d as %q: %v", cmd.Process.Pid, name, err)
336+
} else {
337+
eStat.Group = isolationGroup
326338
}
327-
return cmd.Process,
328-
api.ExecStatus{
329-
State: api.ExecRunning,
330-
Pid: cmd.Process.Pid,
331-
},
332-
runningState
339+
return cmd.Process, eStat, runningState
333340
}
334341

335342
func (c *child) terminate(p *os.Process, s *api.ExecStatus) {
@@ -345,9 +352,32 @@ func (c *child) kill(p *os.Process, s *api.ExecStatus) {
345352
if err := syscall.Kill(-p.Pid, syscall.SIGKILL); err != nil {
346353
log.Printf("failed to kill %d: %v", p.Pid, err)
347354
}
355+
if s.Group != "" {
356+
if err := c.isolator.cleanup(context.TODO(), s.Group); err != nil {
357+
log.Printf("failed to cleanup isolation group %q: %v", s.Group, err)
358+
}
359+
}
348360
s.State = api.ExecStopping
349361
}
350362

363+
func (c *child) cleanup(s *api.ExecStatus) {
364+
if s.Group == "" {
365+
return
366+
}
367+
if err := c.isolator.cleanup(context.TODO(), s.Group); err != nil {
368+
log.Printf("failed to cleanup isolation group %q: %v", s.Group, err)
369+
} else {
370+
s.Group = ""
371+
}
372+
}
373+
374+
func (c *child) cleanupAll(s *api.ChildStatus) {
375+
for i := range s.Init {
376+
c.cleanup(&s.Init[i])
377+
}
378+
c.cleanup(&s.Main)
379+
}
380+
351381
func cloneStatus(s api.ChildStatus) *api.ChildStatus {
352382
r := s
353383
r.Init = slices.Clone(s.Init)

addons/pm/server/child_test.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ func TestChildSleeps(t *testing.T) {
1717
t.SkipNow() // does not return
1818
}
1919

20+
isolator, err := getIsolator()
21+
require.NoError(t, err)
22+
2023
// run a simple sequence of init containers followed by a process container
2124
def := api.Child{
2225
Name: "sleeps",
@@ -35,7 +38,7 @@ func TestChildSleeps(t *testing.T) {
3538
Args: []string{"1h"}, // we will kill this one
3639
},
3740
}
38-
c := newChild(def)
41+
c := newChild(def, isolator)
3942
// TODO: this will hang if something goes wrong
4043
t.Cleanup(c.Wait)
4144

@@ -78,6 +81,9 @@ func TestChildFails(t *testing.T) {
7881
t.SkipNow() // does not return
7982
}
8083

84+
isolator, err := getIsolator()
85+
require.NoError(t, err)
86+
8187
td := t.TempDir()
8288

8389
// run a simple sequence of init containers followed by a process container
@@ -98,7 +104,7 @@ func TestChildFails(t *testing.T) {
98104
Cwd: td,
99105
},
100106
}
101-
c := newChild(def)
107+
c := newChild(def, isolator)
102108
// speed up the restart timers to make this test not so slow
103109
c.restartDelay = 20 * time.Millisecond
104110

addons/pm/server/daemon.go

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,19 @@ type daemon struct {
2020
children map[string]*child
2121
onTerminate context.CancelFunc
2222
tasks []Task
23+
isolator isolator
2324
}
2425

25-
func NewDaemon(tasks ...Task) *daemon {
26+
func NewDaemon(tasks ...Task) (*daemon, error) {
27+
isolator, err := getIsolator()
28+
if err != nil {
29+
return nil, err
30+
}
2631
return &daemon{
2732
children: make(map[string]*child),
2833
tasks: slices.Clone(tasks),
29-
}
34+
isolator: isolator,
35+
}, nil
3036
}
3137

3238
var _ api.API = (*daemon)(nil)
@@ -86,7 +92,7 @@ func (d *daemon) PutChild(ctx context.Context, child api.Child) (*api.ChildWithS
8692
if _, ok := d.children[child.Name]; ok {
8793
return nil, internal.WithStatus(http.StatusConflict, fmt.Errorf("child %s already exists", child.Name))
8894
}
89-
c := newChild(child)
95+
c := newChild(child, d.isolator)
9096
d.children[child.Name] = c
9197
go func() {
9298
c.run()
@@ -227,9 +233,7 @@ func (d *daemon) Terminate(context.Context) error {
227233
d.mu.Unlock()
228234
var wg sync.WaitGroup
229235
for _, child := range children {
230-
wg.Add(1)
231-
go func() {
232-
defer wg.Done()
236+
wg.Go(func() {
233237
child.cmds <- childStop
234238
// wait for it to stop
235239
// TODO: avoid polling
@@ -242,7 +246,7 @@ func (d *daemon) Terminate(context.Context) error {
242246
}
243247
child.cmds <- childDelete
244248
child.Wait()
245-
}()
249+
})
246250
}
247251
wg.Wait()
248252
log.Print("daemon done")

0 commit comments

Comments
 (0)