projects/guix·blob python-team · refs

529 lines · 24.3 KiB

1;;; GNU Guix --- Functional package management for GNU
2;;; Copyright © 2016 Mathieu Lirzin <[email protected]>
3;;; Copyright © 2016-2026 Ludovic Courtès <[email protected]>
4;;; Copyright © 2017, 2020 Mathieu Othacehe <[email protected]>
5;;; Copyright © 2017 Jan Nieuwenhuizen <[email protected]>
6;;; Copyright © 2018, 2019 Ricardo Wurmus <[email protected]>
7;;; Copyright © 2018 Clément Lassieur <[email protected]>
8;;;
9;;; This file is part of GNU Guix.
10;;;
11;;; GNU Guix is free software; you can redistribute it and/or modify
12;;; it under the terms of the GNU General Public License as published by
13;;; the Free Software Foundation, either version 3 of the License, or
14;;; (at your option) any later version.
15;;;
16;;; GNU Guix is distributed in the hope that it will be useful,
17;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19;;; GNU General Public License for more details.
20;;;
21;;; You should have received a copy of the GNU General Public License
22;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
23
24;;;; Commentary:
25;;;
26;;; This module implements a service that to run instances of Cuirass, a
27;;; continuous integration tool.
28;;;
29;;;; Code:
30
31(define-module (gnu services cuirass)
32 #:use-module (guix channels)
33 #:use-module (guix gexp)
34 #:use-module (guix records)
35 #:use-module (guix store)
36 #:use-module (guix utils)
37 #:use-module (gnu packages admin)
38 #:use-module (gnu packages ci)
39 #:use-module (gnu packages databases)
40 #:use-module (gnu packages version-control)
41 #:use-module (gnu services)
42 #:use-module (gnu services base)
43 #:use-module (gnu services databases)
44 #:use-module (gnu services shepherd)
45 #:use-module (gnu services admin)
46 #:use-module (gnu system shadow)
47 #:use-module (srfi srfi-1)
48 #:use-module (ice-9 match)
49 #:export (cuirass-remote-server-configuration
50 cuirass-remote-server-configuration?
51
52 cuirass-configuration
53 cuirass-configuration?
54 cuirass-service-type
55
56 cuirass-remote-worker-configuration
57 cuirass-remote-worker-configuration?
58 cuirass-remote-worker-service-type))
59
60(define %cuirass-default-database
61 "dbname=cuirass")
62
63(define-record-type* <cuirass-remote-server-configuration>
64 cuirass-remote-server-configuration make-cuirass-remote-server-configuration
65 cuirass-remote-server-configuration?
66 (backend-port cuirass-remote-server-configuration-backend-port ;int
67 (default 5555))
68 (log-port cuirass-remote-server-configuration-log-port ;int
69 (default 5556))
70 (publish-port cuirass-remote-server-configuration-publish-port ;int
71 (default 5557))
72 (log-file cuirass-remote-server-log-file ;string
73 (default "/var/log/cuirass-remote-server.log"))
74 (cache cuirass-remote-server-configuration-cache ;string
75 (default "/var/cache/cuirass/remote/"))
76 (log-expiry cuirass-remote-server-configuration-log-expiry
77 (default (* 6 30 24 3600))) ;6 months
78 (publish? cuirass-remote-server-configuration-publish? ;boolean
79 (default #t))
80 (trigger-url cuirass-remote-server-trigger-url ;string
81 (default #f))
82 (public-key cuirass-remote-server-configuration-public-key ;string
83 (default #f))
84 (private-key cuirass-remote-server-configuration-private-key ;string
85 (default #f)))
86
87(define-record-type* <cuirass-configuration>
88 cuirass-configuration make-cuirass-configuration
89 cuirass-configuration?
90 (cuirass cuirass-configuration-cuirass ;file-like
91 (default cuirass))
92 (log-file cuirass-configuration-log-file ;string
93 (default "/var/log/cuirass.log"))
94 (web-log-file cuirass-configuration-web-log-file ;string
95 (default "/var/log/cuirass-web.log"))
96 (cache-directory cuirass-configuration-cache-directory ;string (dir-name)
97 (default "/var/cache/cuirass"))
98 (user cuirass-configuration-user ;string
99 (default "cuirass"))
100 (group cuirass-configuration-group ;string
101 (default "cuirass"))
102 (interval cuirass-configuration-interval ;integer (seconds)
103 (default 300))
104 (parallel-evaluations cuirass-configuration-parallel-evaluations
105 (default #f)) ;integer | #f
106 (ttl cuirass-configuration-ttl ;integer
107 (default 2592000))
108 (evaluation-ttl cuirass-configuration-evaluation-ttl
109 (default (* 24 30 24 3600))) ;integer (seconds)
110 (build-expiry cuirass-configuration-build-expiry
111 (default (* 4 30 24 3600))) ;integer(seconds)
112 (threads cuirass-configuration-threads ;integer
113 (default #f))
114 (web-threads cuirass-configuration-web-threads ;integer | #f
115 (default #f))
116 (parameters cuirass-configuration-parameters ;string
117 (default #f))
118 (remote-server cuirass-configuration-remote-server
119 (default #f))
120 (database cuirass-configuration-database ;string
121 (default %cuirass-default-database))
122 (port cuirass-configuration-port ;integer (port)
123 (default 8081))
124 (host cuirass-configuration-host ;string
125 (default "localhost"))
126 (specifications cuirass-configuration-specifications)
127 ;gexp that evaluates to specification-alist
128 (one-shot? cuirass-configuration-one-shot? ;boolean
129 (default #f))
130 (fallback? cuirass-configuration-fallback? ;boolean
131 (default #f))
132 (extra-options cuirass-configuration-extra-options
133 (default '()))
134 (web-extra-options cuirass-configuration-web-extra-options
135 (default '())))
136
137(define (cuirass-configuration->specification-file config)
138 "Return a specification file built from CONFIG and validate it."
139 (define cuirass
140 (cuirass-configuration-cuirass config))
141
142 (define build
143 #~(begin
144 (use-modules (ice-9 pretty-print))
145
146 (call-with-output-file #$output
147 (lambda (port)
148 (pretty-print '#$(cuirass-configuration-specifications config)
149 port)))
150
151 ;; Validate the spec file upfront.
152 (unless (zero?
153 (system* #$(file-append cuirass "/bin/cuirass") "register"
154 "--check" "-S" #$output))
155 (exit 1))))
156
157 (computed-file "cuirass-specs.scm" build))
158
159(define (cuirass-shepherd-service config)
160 "Return a <shepherd-service> for the Cuirass service with CONFIG."
161 (define (endpoint name)
162 #~(endpoint (make-socket-address AF_UNIX
163 #$(in-vicinity "/var/run/cuirass" name))
164 #:name #$name
165 #:socket-owner #$(cuirass-configuration-user config)
166 #:socket-group #$(cuirass-configuration-group config)
167 #:socket-directory-permissions #o700))
168
169 (let ((cuirass (cuirass-configuration-cuirass config))
170 (cache-directory (cuirass-configuration-cache-directory config))
171 (web-log-file (cuirass-configuration-web-log-file config))
172 (main-log-file (cuirass-configuration-log-file config))
173 (user (cuirass-configuration-user config))
174 (group (cuirass-configuration-group config))
175 (interval (cuirass-configuration-interval config))
176 (parallel-evaluations (cuirass-configuration-parallel-evaluations config))
177 (ttl (cuirass-configuration-ttl config))
178 (evaluation-ttl (cuirass-configuration-evaluation-ttl config))
179 (build-expiry (cuirass-configuration-build-expiry config))
180 (threads (cuirass-configuration-threads config))
181 (web-threads (cuirass-configuration-web-threads config))
182 (parameters (cuirass-configuration-parameters config))
183 (remote-server (cuirass-configuration-remote-server config))
184 (database (cuirass-configuration-database config))
185 (port (cuirass-configuration-port config))
186 (host (cuirass-configuration-host config))
187 (config-file (cuirass-configuration->specification-file config))
188 (one-shot? (cuirass-configuration-one-shot? config))
189 (fallback? (cuirass-configuration-fallback? config))
190 (extra-options (cuirass-configuration-extra-options config))
191 (web-extra-options (cuirass-configuration-web-extra-options config)))
192 `(,(shepherd-service
193 (documentation "Run Cuirass.")
194 (provision '(cuirass))
195 (requirement '(user-processes
196 guix-daemon
197 postgres postgres-roles networking))
198 (start #~(make-systemd-constructor
199 (list (string-append #$cuirass "/bin/cuirass")
200 "register"
201 "--cache-directory" #$cache-directory
202 "--specifications" #$config-file
203 "--database" #$database
204 "--interval" #$(number->string interval)
205 #$@(if parallel-evaluations
206 (list (string-append
207 "--parallel-evaluations="
208 (number->string evaluation-ttl)))
209 '())
210 #$@(if ttl
211 (list (string-append
212 "--ttl="
213 (number->string ttl)
214 "s"))
215 '())
216 #$@(if evaluation-ttl
217 (list (string-append
218 "--evaluation-ttl="
219 (number->string evaluation-ttl)
220 "s"))
221 '())
222 #$@(if build-expiry
223 (list (string-append
224 "--build-expiry="
225 (number->string build-expiry)
226 "s"))
227 '())
228 #$@(if threads
229 (list (string-append
230 "--threads="
231 (number->string threads)))
232 '())
233 #$@(if parameters
234 (list #~(string-append "--parameters="
235 #$parameters))
236 '())
237 #$@(if remote-server '("--build-remote") '())
238 #$@(if one-shot? '("--one-shot") '())
239 #$@(if fallback? '("--fallback") '())
240 #$@extra-options)
241
242 ;; Unix-domain sockets that trigger socket activation.
243 (list #$(endpoint "bridge")
244 #$(endpoint "remote-builds"))
245
246 ;; Start working right away.
247 #:lazy-start? #f
248
249 #:environment-variables
250 (list "LC_ALL=C.UTF-8" ;for proper file name decoding
251 "GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt"
252 "COLUMNS=200" ;for backtraces
253 (string-append "GIT_EXEC_PATH=" #$git
254 "/libexec/git-core"))
255
256 #:user #$user
257 #:group #$group
258 #:log-file #$main-log-file))
259 (stop #~(make-systemd-destructor))
260 (actions (list (shepherd-configuration-action config-file))))
261 ,(shepherd-service
262 (documentation "Run Cuirass web interface.")
263 (provision '(cuirass-web))
264 (requirement '(user-processes cuirass))
265
266 ;; XXX: Respawn slowly to work around the fact that 'cuirass' is not
267 ;; ready yet (not listening to the bridge) when it gets in 'started'
268 ;; state.
269 (respawn-delay 1)
270 (respawn-limit #~'(10 . 15))
271
272 (start #~(make-forkexec-constructor
273 (list (string-append #$cuirass "/bin/cuirass")
274 "web"
275 "--database" #$database
276 "--listen" #$host
277 "--port" #$(number->string port)
278 #$@(if web-threads
279 (list #~(string-append "--threads="
280 #$web-threads))
281 '())
282 #$@(if parameters
283 (list #~(string-append "--parameters="
284 #$parameters))
285 '())
286 #$@web-extra-options)
287
288 #:user #$user
289 #:group #$group
290 #:log-file #$web-log-file))
291 (stop #~(make-kill-destructor)))
292 ,@(if remote-server
293 (match-record remote-server <cuirass-remote-server-configuration>
294 (backend-port publish-port log-file log-expiry cache publish?
295 trigger-url public-key private-key)
296 (list
297 (shepherd-service
298 (documentation "Run Cuirass remote build server.")
299 (provision '(cuirass-remote-server))
300 (requirement '(user-processes avahi-daemon cuirass))
301 (start #~(make-forkexec-constructor
302 (list (string-append #$cuirass "/bin/cuirass")
303 "remote-server"
304 (string-append "--database=" #$database)
305 (string-append "--cache=" #$cache)
306 (string-append "--user=" #$user)
307 (string-append "--log-expiry="
308 #$(number->string log-expiry)
309 "s")
310 #$@(if backend-port
311 (list (string-append
312 "--backend-port="
313 (number->string backend-port)))
314 '())
315 #$@(if publish-port
316 (list (string-append
317 "--publish-port="
318 (number->string publish-port)))
319 '())
320 #$@(if parameters
321 (list #~(string-append "--parameters="
322 #$parameters))
323 '())
324 #$@(if trigger-url
325 (list
326 (string-append
327 "--trigger-substitute-url="
328 trigger-url))
329 '())
330 #$@(if publish?
331 '()
332 (list "--no-publish"))
333 #$@(if public-key
334 (list
335 (string-append "--public-key="
336 public-key))
337 '())
338 #$@(if private-key
339 (list
340 (string-append "--private-key="
341 private-key))
342 '()))
343 #:log-file #$log-file))
344 (stop #~(make-kill-destructor)))))
345 '()))))
346
347(define (cuirass-account config)
348 "Return the user accounts and user groups for CONFIG."
349 (let ((cuirass-user (cuirass-configuration-user config))
350 (cuirass-group (cuirass-configuration-group config)))
351 (list (user-group
352 (name cuirass-group)
353 (system? #t))
354 (user-account
355 (name cuirass-user)
356 (group cuirass-group)
357 (system? #t)
358 (comment "Cuirass privilege separation user")
359 (home-directory (string-append "/var/lib/" cuirass-user))
360 (shell (file-append shadow "/sbin/nologin"))))))
361
362(define (cuirass-postgresql-role config)
363 (let ((user (cuirass-configuration-user config)))
364 (list (postgresql-role
365 (name user)
366 (create-database? #t)))))
367
368(define (cuirass-activation config)
369 "Return the activation code for CONFIG."
370 (let* ((cache (cuirass-configuration-cache-directory config))
371 (remote-server (cuirass-configuration-remote-server config))
372 (remote-cache (and remote-server
373 (cuirass-remote-server-configuration-cache
374 remote-server)))
375 (user (cuirass-configuration-user config))
376 ;; RUNSTATEDIR contains the "bridge" Unix-domain socket that 'cuirass
377 ;; web' connects to to communicate with 'cuirass register'.
378 (runstatedir "/var/run/cuirass")
379 (log "/var/log/cuirass")
380 (profile (string-append "/var/guix/profiles/per-user/" user))
381 (roots (string-append profile "/cuirass"))
382 (group (cuirass-configuration-group config)))
383 (with-imported-modules '((guix build utils))
384 #~(begin
385 (use-modules (guix build utils))
386
387 (mkdir-p #$cache)
388 (mkdir-p #$log)
389 (mkdir-p #$roots)
390 (mkdir-p #$runstatedir)
391
392 (when #$remote-cache
393 (mkdir-p #$remote-cache))
394
395 (let ((uid (passwd:uid (getpw #$user)))
396 (gid (group:gid (getgr #$group))))
397 (chown #$cache uid gid)
398 (chown #$log uid gid)
399 (chown #$roots uid gid)
400 (chown #$profile uid gid)
401 (chown #$runstatedir uid gid)
402 (chmod #$runstatedir #o700)
403
404 (when #$remote-cache
405 (chown #$remote-cache uid gid)))))))
406
407(define cuirass-service-type
408 (service-type
409 (name 'cuirass)
410 (extensions
411 (list
412 (service-extension profile-service-type ;for 'info cuirass'
413 (compose list cuirass-configuration-cuirass))
414 (service-extension activation-service-type cuirass-activation)
415 (service-extension shepherd-root-service-type cuirass-shepherd-service)
416 (service-extension account-service-type cuirass-account)
417 ;; Make sure postgresql and postgresql-role are instantiated.
418 (service-extension postgresql-service-type (const #t))
419 (service-extension postgresql-role-service-type
420 cuirass-postgresql-role)))
421 (description
422 "Run the Cuirass continuous integration service.")))
423
424(define-record-type* <cuirass-remote-worker-configuration>
425 cuirass-remote-worker-configuration make-cuirass-remote-worker-configuration
426 cuirass-remote-worker-configuration?
427 (cuirass cuirass-remote-worker-configuration-cuirass ;file-like
428 (default cuirass))
429 (workers cuirass-remote-worker-workers ;int
430 (default 1))
431 (server cuirass-remote-worker-server ;string
432 (default #f))
433 (systems cuirass-remote-worker-systems ;list
434 (default (list (%current-system))))
435 (minimum-disk-space cuirass-remote-worker-minimum-disk-space
436 (default 5))
437 (log-file cuirass-remote-worker-log-file ;string
438 (default "/var/log/cuirass-remote-worker.log"))
439 (publish-port cuirass-remote-worker-configuration-publish-port ;int
440 (default 5558))
441 (substitute-urls cuirass-remote-worker-configuration-substitute-urls
442 (default %default-substitute-urls)) ;list of strings
443 (public-key cuirass-remote-worker-configuration-public-key ;string
444 (default #f))
445 (private-key cuirass-remote-worker-configuration-private-key ;string
446 (default #f)))
447
448(define %cuirass-remote-worker-accounts
449 ;; User account and group for the 'cuirass remote-worker' process.
450 (list (user-group
451 (name "cuirass-worker")
452 (system? #t))
453 (user-account
454 (name "cuirass-worker")
455 (group name)
456 (system? #t)
457 (comment "Cuirass worker privilege separation user")
458 (home-directory "/var/empty")
459 (shell (file-append shadow "/sbin/nologin")))))
460
461(define (cuirass-remote-worker-shepherd-service config)
462 "Return a <shepherd-service> for the Cuirass remote worker service with
463CONFIG."
464 (match-record config <cuirass-remote-worker-configuration>
465 (cuirass workers server systems log-file publish-port
466 minimum-disk-space
467 substitute-urls public-key private-key)
468 (list (shepherd-service
469 (documentation "Run Cuirass remote build worker.")
470 (provision '(cuirass-remote-worker))
471 (requirement '(user-processes avahi-daemon guix-daemon networking))
472 (start #~(make-forkexec-constructor
473 (list (string-append #$cuirass "/bin/cuirass")
474 "remote-worker"
475 "--user=cuirass-worker" ;drop privileges early on
476 (string-append "--workers="
477 #$(number->string workers))
478 (string-append "--minimum-disk-space="
479 #$(number->string minimum-disk-space))
480 #$@(if server
481 (list (string-append "--server=" server))
482 '())
483 #$@(if systems
484 (list (string-append
485 "--systems="
486 (string-join systems ",")))
487 '())
488 #$@(if publish-port
489 (list (string-append
490 "--publish-port="
491 (number->string publish-port)))
492 '())
493 #$@(if substitute-urls
494 (list (string-append
495 "--substitute-urls="
496 (string-join substitute-urls)))
497 '())
498 #$@(if public-key
499 (list
500 (string-append "--public-key="
501 public-key))
502 '())
503 #$@(if private-key
504 (list
505 (string-append "--private-key="
506 private-key))
507 '()))
508
509 ;; Work around
510 ;; <https://codeberg.org/guile/fibers/issues/83>.
511 ;; TODO: Remove when using Guile > 3.0.9.
512 #:environment-variables
513 (if (string-prefix? "aarch64-" %host-type)
514 '("GUILE_JIT_THRESHOLD=-1")
515 '())
516
517 #:log-file #$log-file))
518 (stop #~(make-kill-destructor))))))
519
520(define cuirass-remote-worker-service-type
521 (service-type
522 (name 'cuirass-remote-worker)
523 (extensions
524 (list (service-extension shepherd-root-service-type
525 cuirass-remote-worker-shepherd-service)
526 (service-extension account-service-type
527 (const %cuirass-remote-worker-accounts))))
528 (description
529 "Run the Cuirass remote build worker service.")))