projects/guix·blob master · refs

570 lines · 21.8 KiB

1;;; GNU Guix --- Functional package management for GNU
2;;; Copyright © 2014 David Thompson <[email protected]>
3;;; Copyright © 2016 Ricardo Wurmus <[email protected]>
4;;; Copyright © 2019 Maxim Cournoyer <[email protected]>
5;;; Copyright © 2021 Xinglu Chen <[email protected]>
6;;; Copyright © 2022 Vivien Kraus <[email protected]>
7;;;
8;;; This file is part of GNU Guix.
9;;;
10;;; GNU Guix is free software; you can redistribute it and/or modify it
11;;; under the terms of the GNU General Public License as published by
12;;; the Free Software Foundation; either version 3 of the License, or (at
13;;; your option) any later version.
14;;;
15;;; GNU Guix is distributed in the hope that it will be useful, but
16;;; WITHOUT ANY WARRANTY; without even the implied warranty of
17;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18;;; GNU General Public License for more details.
19;;;
20;;; You should have received a copy of the GNU General Public License
21;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
22
23(define-module (test-pypi)
24 #:use-module (guix import pypi)
25 #:use-module (guix base32)
26 #:use-module (guix memoization)
27 #:use-module (guix utils)
28 #:use-module ((guix base16) #:select (base16-string->bytevector))
29 #:use-module (guix upstream)
30 #:use-module (gcrypt hash)
31 #:use-module (guix tests)
32 #:use-module (guix tests http)
33 #:use-module ((guix download) #:select (url-fetch))
34 #:use-module (guix build-system pyproject)
35 #:use-module ((guix build utils)
36 #:select (delete-file-recursively
37 which mkdir-p dump-port
38 with-directory-excursion))
39 #:use-module ((guix diagnostics) #:select (guix-warning-port))
40 #:use-module ((guix build syscalls) #:select (mkdtemp!))
41 #:use-module (json)
42 #:use-module (srfi srfi-26)
43 #:use-module (srfi srfi-34)
44 #:use-module (srfi srfi-35)
45 #:use-module (srfi srfi-64)
46 #:use-module (ice-9 match)
47 #:use-module (ice-9 optargs))
48
49(define default-sha256
50 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
51(define default-sha256/base32
52 (bytevector->nix-base32-string
53 (base16-string->bytevector default-sha256)))
54
55(define* (foo-json #:key (name "foo") (name-in-url #f))
56 "Create a JSON description of an example pypi package, named @var{name},
57optionally using a different @var{name in its URL}."
58 (scm->json-string
59 `((info
60 . ((version . "1.0.0")
61 (name . ,name)
62 (license . "GNU LGPL")
63 (summary . "summary")
64 (home_page . "http://example.com")
65 (classifiers . #())
66 (download_url . "")))
67 (urls . #())
68 (releases
69 . ((1.0.0
70 . #(((url . ,(format #f "~a/~a-1.0.0.egg"
71 (%local-url #:path "")
72 (or name-in-url name)))
73 (packagetype . "bdist_egg"))
74 ((url . ,(format #f "~a/~a-1.0.0.tar.gz"
75 (%local-url #:path "")
76 (or name-in-url name)))
77 (packagetype . "sdist")
78 (digests . (("sha256" . ,default-sha256))))
79 ((url . ,(format #f "~a/~a-1.0.0-py2.py3-none-any.whl"
80 (%local-url #:path "")
81 (or name-in-url name)))
82 (packagetype . "bdist_wheel")))))))))
83
84(define test-specifications
85 '("Fizzy [foo, bar]"
86 "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1"
87 "SomethingWithMarker[foo]>1.0;python_version<\"2.7\""
88 "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\""
89 "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\
90sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
91
92(define test-requires.txt "\
93# A comment
94 # A comment after a space
95foo ~= 3
96bar != 2
97
98[test]
99pytest (>=2.5.0)
100pytest-cov # read but ignored
101")
102
103;; Beaker contains only optional dependencies.
104(define test-requires.txt-beaker "\
105[crypto]
106pycryptopp>=0.5.12
107
108[cryptography]
109cryptography
110
111[testsuite]
112Mock
113coverage
114")
115
116(define test-pyproject.toml "\
117[build-system]
118requires = [\"dummy-build-dep-a\", \"dummy-build-dep-b\"]
119
120[project]
121dependencies = [
122 \"dummy-dep-a\",
123 \"dummy-dep-b\",
124]
125
126[project.optional-dependencies]
127test = [\"dummy-test-dep-a\", \"dummy-test-dep-b\"]
128")
129
130(define test-metadata "\
131Classifier: Programming Language :: Python :: 3.7
132Requires-Dist: baz ~= 3
133Requires-Dist: bar != 2
134Provides-Extra: test
135Requires-Dist: pytest (>=2.5.0) ; extra == 'test'
136")
137
138(define test-metadata-with-extras "
139Classifier: Programming Language :: Python :: 3.7
140Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
141Requires-Dist: wrapt (<2,>=1)
142Requires-Dist: bar
143
144Provides-Extra: dev
145Requires-Dist: tox ; extra == 'dev'
146Requires-Dist: bumpversion (<1) ; extra == 'dev'
147")
148
149;;; Provides-Extra can appear before Requires-Dist.
150(define test-metadata-with-extras-jedi "\
151Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
152Provides-Extra: testing
153Requires-Dist: parso (>=0.3.0)
154Provides-Extra: testing
155Requires-Dist: pytest (>=3.1.0); extra == 'testing'
156")
157
158(define sample-directory
159 ;; Directory containing tarballs and .whl files for this test.
160 (let ((template (string-append (or (getenv "TMPDIR") "/tmp")
161 "/guix-pypi-test-XXXXXX")))
162 (mkdtemp! template)))
163
164(define (pypi-tarball name specs)
165 "Return a PyPI tarball called NAME suffixed with '.tar.gz' and containing
166the files specified in SPECS. Return its file name."
167 (let ((directory (in-vicinity sample-directory name))
168 (tarball (in-vicinity sample-directory (string-append name ".tar.gz"))))
169 (false-if-exception (delete-file tarball))
170 (mkdir-p directory)
171 (for-each (match-lambda
172 ((file content)
173 (mkdir-p (in-vicinity directory (dirname file)))
174 (call-with-output-file (in-vicinity directory file)
175 (lambda (port)
176 (display content port)))))
177 specs)
178 (parameterize ((current-output-port (%make-void-port "w0")))
179 (system* "tar" "-C" sample-directory "-czvf" tarball
180 (basename directory)))
181 (delete-file-recursively directory)
182 tarball))
183
184(define (wheel-file name specs)
185 "Return a Wheel file called NAME suffixed with '.whl' and containing the
186files specified by SPECS. Return its file name."
187 (let* ((directory (in-vicinity sample-directory
188 (string-append name ".dist-info")))
189 (zip-file (in-vicinity sample-directory
190 (string-append name ".zip")))
191 (whl-file (in-vicinity sample-directory
192 (string-append name ".whl"))))
193 (false-if-exception (delete-file whl-file))
194 (mkdir-p directory)
195 (for-each (match-lambda
196 ((file content)
197 (mkdir-p (in-vicinity directory (dirname file)))
198 (call-with-output-file (in-vicinity directory file)
199 (lambda (port)
200 (display content port)))))
201 specs)
202 ;; zip always adds a "zip" extension to the file it creates,
203 ;; so we need to rename it.
204 (with-directory-excursion (dirname directory)
205 (system* "zip" "-qr" zip-file (basename directory)))
206 (rename-file zip-file whl-file)
207 (delete-file-recursively directory)
208 whl-file))
209
210(define (file-dump file)
211 "Return a procedure that dumps FILE to the given port."
212 (lambda (output)
213 (call-with-input-file file
214 (lambda (input)
215 (dump-port input output)))))
216
217(define-syntax-rule (with-pypi responses body ...)
218 (with-http-server responses
219 (parameterize ((%pypi-base-url (%local-url #:path "/")))
220 body ...)))
221
222
223(test-begin "pypi")
224
225(test-equal "guix-package->pypi-name, old URL style"
226 "psutil"
227 (guix-package->pypi-name
228 (dummy-package "foo"
229 (source (dummy-origin
230 (uri
231 "https://pypi.org/packages/source/p/psutil/psutil-4.3.0.tar.gz"))))))
232
233(test-equal "guix-package->pypi-name, new URL style"
234 "certbot"
235 (guix-package->pypi-name
236 (dummy-package "foo"
237 (source (dummy-origin
238 (uri
239 "https://pypi.org/packages/a2/3b/4756e6a0ceb14e084042a2a65c615d68d25621c6fd446d0fc10d14c4ce7d/certbot-0.8.1.tar.gz"))))))
240
241(test-equal "guix-package->pypi-name, several URLs"
242 "cram"
243 (guix-package->pypi-name
244 (dummy-package "foo"
245 (source
246 (dummy-origin
247 (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
248 (pypi-uri "cram" "0.7"))))))))
249
250(test-equal "guix-package->pypi-name, honor 'upstream-name'"
251 "bar-3"
252 (guix-package->pypi-name
253 (dummy-package "foo"
254 (properties
255 '((upstream-name . "bar-3"))))))
256
257(test-equal "specification->requirement-name"
258 '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
259 (map specification->requirement-name test-specifications))
260
261(test-equal "parse-requires.txt"
262 (list '("foo" "bar") '("pytest" "pytest-cov"))
263 (mock ((ice-9 ports) call-with-input-file
264 call-with-input-string)
265 (parse-requires.txt test-requires.txt)))
266
267(test-equal "parse-requires.txt - Beaker"
268 (list '() '("Mock" "coverage"))
269 (mock ((ice-9 ports) call-with-input-file
270 call-with-input-string)
271 (parse-requires.txt test-requires.txt-beaker)))
272
273(test-equal "parse-wheel-metadata"
274 (list '("baz" "bar") '("pytest"))
275 (mock ((ice-9 ports) call-with-input-file
276 call-with-input-string)
277 (parse-wheel-metadata test-metadata)))
278
279(test-equal "parse-wheel-metadata, with extras"
280 (list '("wrapt" "bar") '("tox" "bumpversion"))
281 (mock ((ice-9 ports) call-with-input-file
282 call-with-input-string)
283 (parse-wheel-metadata test-metadata-with-extras)))
284
285(test-equal "parse-wheel-metadata, with extras - Jedi"
286 (list '("parso") '("pytest"))
287 (mock ((ice-9 ports) call-with-input-file
288 call-with-input-string)
289 (parse-wheel-metadata test-metadata-with-extras-jedi)))
290
291(test-equal "find-project-url, with numpy"
292 "numpy"
293 (find-project-url
294 "numpy"
295 "https://files.pythonhosted.org/packages/0a/c8/a62767a6b374a0dfb02d2a0456e5f56a372cdd1689dbc6ffb6bf1ddedbc0/numpy-1.22.1.zip"))
296
297(test-equal "find-project-url, uWSGI"
298 "uwsgi"
299 (find-project-url
300 "uWSGI"
301 "https://files.pythonhosted.org/packages/24/fd/93851e4a076719199868d4c918cc93a52742e68370188c1c570a6e42a54f/uwsgi-2.0.20.tar.gz"))
302
303(test-equal "find-project-url, flake8-array-spacing"
304 "flake8_array_spacing"
305 (find-project-url
306 "flake8-array-spacing"
307 "https://files.pythonhosted.org/packages/a4/21/ff29b901128b681b7de7a2787b3aeb3e1f3cba4a8c0cffa9712cbff016bc/flake8_array_spacing-0.2.0.tar.gz"))
308
309(test-equal "find-project-url, foo/goo"
310 "foo"
311 (find-project-url
312 "foo"
313 "https://files.pythonhosted.org/packages/f0/f00/goo-0.0.0.tar.gz"))
314
315(test-assert "pypi->guix-package, no wheel"
316 (let ((tarball (pypi-tarball
317 "foo-1.0.0"
318 `(("src/bizarre.egg-info/requires.txt"
319 ,test-requires.txt))))
320 (twice (lambda (lst) (append lst lst))))
321 (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
322 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
323 ("/foo/json" 200 ,(lambda (port)
324 (display (foo-json) port)))))
325 (match (pypi->guix-package "foo")
326 (`(package
327 (name "python-foo")
328 (version "1.0.0")
329 (source (origin
330 (method url-fetch)
331 (uri (pypi-uri "foo" version))
332 (sha256
333 (base32 ,(? string? hash)))))
334 (build-system pyproject-build-system)
335 (propagated-inputs (list python-bar python-foo))
336 (native-inputs (list python-pytest
337 python-setuptools))
338 (home-page "https://example.com")
339 (synopsis "summary")
340 (description "summary.")
341 (license license:lgpl2.0))
342 (and (string=? default-sha256/base32 hash)
343 (equal? (pypi->guix-package "foo" #:version "1.0.0")
344 (pypi->guix-package "foo"))
345 (guard (c ((error? c) #t))
346 (pypi->guix-package "foo" #:version "42"))))
347 (x
348 (pk 'fail x #f))))))
349
350(test-assert "pypi->guix-package, no wheel, no requires.txt, but pyproject.toml"
351 (let ((tarball (pypi-tarball
352 "foo-1.0.0"
353 `(("pyproject.toml" ,test-pyproject.toml))))
354 (twice (lambda (lst) (append lst lst))))
355 (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
356 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
357 ("/foo/json" 200 ,(lambda (port)
358 (display (foo-json) port)))))
359 ;; Not clearing the memoization cache here would mean returning the value
360 ;; computed in the previous test.
361 (invalidate-memoization! pypi->guix-package)
362 (match (pypi->guix-package "foo")
363 (`(package
364 (name "python-foo")
365 (version "1.0.0")
366 (source (origin
367 (method url-fetch)
368 (uri (pypi-uri "foo" version))
369 (sha256
370 (base32 ,(? string? hash)))))
371 (build-system pyproject-build-system)
372 (propagated-inputs (list python-dummy-dep-a python-dummy-dep-b))
373 (native-inputs (list python-dummy-build-dep-a python-dummy-build-dep-b
374 python-dummy-test-dep-a python-dummy-test-dep-b))
375 (home-page "https://example.com")
376 (synopsis "summary")
377 (description "summary.")
378 (license license:lgpl2.0))
379 (and (string=? default-sha256/base32 hash)
380 (equal? (pypi->guix-package "foo" #:version "1.0.0")
381 (pypi->guix-package "foo"))
382 (guard (c ((error? c) #t))
383 (pypi->guix-package "foo" #:version "42"))))
384 (x
385 (pk 'fail x #f))))))
386
387(test-assert "pypi->guix-package, no wheel, but requires.txt and pyproject.toml"
388 (let ((tarball (pypi-tarball
389 "foo-1.0.0"
390 `(("foo-1.0.0/pyproject.toml" ,test-pyproject.toml)
391 ("foo-1.0.0/bizarre.egg-info/requires.txt"
392 ,test-requires.txt))))
393 (twice (lambda (lst) (append lst lst))))
394 (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
395 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
396 ("/foo/json" 200 ,(lambda (port)
397 (display (foo-json) port)))))
398 ;; Not clearing the memoization cache here would mean returning the value
399 ;; computed in the previous test.
400 (invalidate-memoization! pypi->guix-package)
401 (match (pypi->guix-package "foo")
402 (`(package
403 (name "python-foo")
404 (version "1.0.0")
405 (source (origin
406 (method url-fetch)
407 (uri (pypi-uri "foo" version))
408 (sha256
409 (base32 ,(? string? hash)))))
410 (build-system pyproject-build-system)
411 ;; Information from requires.txt and pyproject.toml is combined.
412 (propagated-inputs (list python-bar python-dummy-dep-a python-dummy-dep-b
413 python-foo))
414 (native-inputs (list python-dummy-build-dep-a python-dummy-build-dep-b
415 python-dummy-test-dep-a python-dummy-test-dep-b
416 python-pytest))
417 (home-page "https://example.com")
418 (synopsis "summary")
419 (description "summary.")
420 (license license:lgpl2.0))
421 (and (string=? default-sha256/base32 hash)
422 (equal? (pypi->guix-package "foo" #:version "1.0.0")
423 (pypi->guix-package "foo"))
424 (guard (c ((error? c) #t))
425 (pypi->guix-package "foo" #:version "42"))))
426 (x
427 (pk 'fail x #f))))))
428
429(test-skip (if (which "zip") 0 1))
430(test-assert "pypi->guix-package, no requires.txt, but wheel."
431 (let ((tarball (pypi-tarball
432 "foo-1.0.0"
433 '(("foo-1.0.0/foo.egg-info/.empty" ""))))
434 (wheel (wheel-file "foo-1.0.0"
435 `(("METADATA" ,test-metadata)))))
436 (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
437 ("/foo-1.0.0-py2.py3-none-any.whl"
438 200 ,(file-dump wheel))
439 ("/foo/json" 200 ,(cut display (foo-json) <>)))
440 ;; Not clearing the memoization cache here would mean returning the value
441 ;; computed in the previous test.
442 (invalidate-memoization! pypi->guix-package)
443 (match (pypi->guix-package "foo")
444 (`(package
445 (name "python-foo")
446 (version "1.0.0")
447 (source (origin
448 (method url-fetch)
449 (uri (pypi-uri "foo" version))
450 (sha256
451 (base32 ,(? string? hash)))))
452 (build-system pyproject-build-system)
453 (propagated-inputs (list python-bar python-baz))
454 (native-inputs (list python-pytest
455 python-setuptools))
456 (home-page "https://example.com")
457 (synopsis "summary")
458 (description "summary.")
459 (license license:lgpl2.0))
460 (string=? default-sha256/base32 hash))
461 (x
462 (pk 'fail x #f))))))
463
464(test-assert "pypi->guix-package, no usable requirement file, no wheel."
465 (let ((tarball (pypi-tarball "foo-1.0.0"
466 '(("foo.egg-info/.empty" "")))))
467 (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
468 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
469 ("/foo/json" 200 ,(lambda (port)
470 (display (foo-json) port))))
471 ;; Not clearing the memoization cache here would mean returning the
472 ;; value computed in the previous test.
473 (invalidate-memoization! pypi->guix-package)
474 (match (pypi->guix-package "foo")
475 (`(package
476 (name "python-foo")
477 (version "1.0.0")
478 (source (origin
479 (method url-fetch)
480 (uri (pypi-uri "foo" version))
481 (sha256
482 (base32 ,(? string? hash)))))
483 (build-system pyproject-build-system)
484 (native-inputs (list python-setuptools))
485 (home-page "https://example.com")
486 (synopsis "summary")
487 (description "summary.")
488 (license license:lgpl2.0))
489 (string=? default-sha256/base32 hash))
490 (x
491 (pk 'fail x #f))))))
492
493(test-assert "pypi->guix-package, package name contains \"-\" followed by digits"
494 (let ((tarball (pypi-tarball "foo-99-1.0.0"
495 `(("src/bizarre.egg-info/requires.txt"
496 ,test-requires.txt)))))
497 (with-pypi `(("/foo-99-1.0.0.tar.gz" 200 ,(file-dump tarball))
498 ("/foo-99-1.0.0-py2.py3-none-any.whl" 404 "")
499 ("/foo-99/json" 200 ,(lambda (port)
500 (display (foo-json #:name "foo-99")
501 port))))
502 (match (pypi->guix-package "foo-99")
503 (`(package
504 (name "python-foo-99")
505 (version "1.0.0")
506 (source (origin
507 (method url-fetch)
508 (uri (pypi-uri "foo-99" version))
509 (sha256
510 (base32 ,(? string? hash)))))
511 (properties (quote (("upstream-name" . "foo-99"))))
512 (build-system pyproject-build-system)
513 (propagated-inputs (list python-bar python-foo))
514 (native-inputs (list python-pytest
515 python-setuptools))
516 (home-page "https://example.com")
517 (synopsis "summary")
518 (description "summary.")
519 (license license:lgpl2.0))
520 (string=? default-sha256/base32 hash))
521 (x
522 (pk 'fail x #f))))))
523
524(test-equal "package-latest-release"
525 (list '("foo-1.0.0.tar.gz")
526 '("foo-1.0.0.tar.gz.asc")
527 (list (upstream-input
528 (name "bar")
529 (downstream-name "python-bar")
530 (type 'propagated))
531 (upstream-input
532 (name "foo")
533 (downstream-name "python-foo")
534 (type 'propagated))
535 (upstream-input
536 (name "pytest")
537 (downstream-name "python-pytest")
538 (type 'native))
539 (upstream-input
540 (name "setuptools")
541 (downstream-name "python-setuptools")
542 (type 'native))))
543 (let ((tarball (pypi-tarball
544 "foo-1.0.0"
545 `(("src/bizarre.egg-info/requires.txt"
546 ,test-requires.txt)))))
547 (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
548 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
549 ("/foo/json" 200 ,(lambda (port)
550 (display (foo-json) port))))
551 (define source
552 (package-latest-release
553 (dummy-package "python-foo"
554 (version "0.1.2")
555 (source (dummy-origin
556 (method url-fetch)
557 (uri (pypi-uri "foo" version))))
558 (build-system pyproject-build-system))
559 (list %pypi-updater)))
560
561 (list (map basename (upstream-source-urls source))
562 (map basename (upstream-source-signature-urls source))
563 (upstream-source-inputs source)))))
564
565(test-end "pypi")
566(delete-file-recursively sample-directory)
567
568;; Local Variables:
569;; eval: (put 'with-pypi 'scheme-indent-function 1)
570;; End: