guix/tests/substitute.scm

;;; GNU Guix --- Functional package management for GNU
;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org>
;;; Copyright © 2014-2015, 2017-2019, 2021-2023 Ludovic Courtès <ludo@gnu.org>
;;;
;;; This file is part of GNU Guix.
;;;
;;; GNU Guix is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; GNU Guix is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.

(define-module (test-substitute)
  #:use-module (guix scripts substitute)
  #:use-module (guix narinfo)
  #:use-module (guix base64)
  #:use-module (gcrypt hash)
  #:use-module (guix serialization)
  #:use-module (gcrypt pk-crypto)
  #:use-module (guix pki)
  #:use-module (guix config)
  #:use-module (guix base32)
  #:use-module ((guix store) #:select (%store-prefix))
  #:use-module ((guix ui) #:select (guix-warning-port))
  #:use-module ((guix utils)
                #:select (call-with-temporary-directory
                          call-with-compressed-output-port))
  #:use-module ((guix build utils)
                #:select (mkdir-p delete-file-recursively dump-port))
  #:use-module (guix tests http)
  #:use-module (rnrs bytevectors)
  #:use-module (rnrs io ports)
  #:use-module (web uri)
  #:use-module (ice-9 regex)
  #:use-module (srfi srfi-11)
  #:use-module (srfi srfi-26)
  #:use-module (srfi srfi-34)
  #:use-module (srfi srfi-35)
  #:use-module ((srfi srfi-64) #:hide (test-error)))

(define-syntax-rule (test-quit name error-rx exp)
  "Emit a test that passes when EXP throws to 'quit' with value 1, and when
it writes to GUIX-WARNING-PORT a messages that matches ERROR-RX."
  (test-equal name
    '(1 #t)
    (let ((error-output (open-output-string)))
      (parameterize ((current-error-port error-output)
                     (guix-warning-port error-output))
        (catch 'quit
          (lambda ()
            exp
            #f)
          (lambda (key value)
            (list value
                  (let ((message (get-output-string error-output)))
                    (->bool (string-match error-rx message))))))))))

(define (request-substitution item destination)
  "Run 'guix substitute --substitute' to fetch ITEM to DESTINATION."
  (false-if-exception (delete-file destination))
  (with-input-from-string (string-append "substitute " item " "
                                         destination "\n")
    (lambda ()
      (guix-substitute "--substitute"))))

(define %public-key
  ;; This key is known to be in the ACL by default.
  (call-with-input-file (string-append %config-directory "/signing-key.pub")
    (compose string->canonical-sexp get-string-all)))

(define %private-key
  (call-with-input-file (string-append %config-directory "/signing-key.sec")
    (compose string->canonical-sexp get-string-all)))

(define* (signature-body bv #:key (public-key %public-key))
  "Return the signature of BV as the base64-encoded body of a narinfo's
'Signature' field."
  (base64-encode
   (string->utf8
    (canonical-sexp->string
     (signature-sexp (bytevector->hash-data (sha256 bv)
                                            #:key-type 'rsa)
                     %private-key
                     public-key)))))

(define %wrong-public-key
  (string->canonical-sexp "(public-key
 (rsa
  (n #00E05873AC2B168760343145918E954EE9AB73C026355693B192E01EE835261AA689E9EF46642E895BCD65C648524059FC450E4BA77A68F4C52D0E39EF0CC9359709AB6AAB153B63782201871325B0FDA19CB401CD99FD0C31A91CA9000AA90A77E82B89E036FB63BC1D3961207469B3B12468977148D376F8012BB12A4B11A8F1#)
  (e #010001#)
  )
 )"))

(define* (signature-field bv-or-str
                          #:key (version "1") (public-key %public-key))
  "Return the 'Signature' field value of bytevector/string BV-OR-STR, using
PUBLIC-KEY as the signature's principal, and using VERSION as the signature
version identifier.."
  (string-append version ";example.gnu.org;"
                 (signature-body (if (string? bv-or-str)
                                     (string->utf8 bv-or-str)
                                     bv-or-str)
                                 #:public-key public-key)))


(test-begin "substitute")

(test-quit "not a number"
    "signature version"
  (narinfo-signature->canonical-sexp
   (signature-field "foo" #:version "not a number")))

(test-quit "wrong version number"
    "unsupported.*version"
  (narinfo-signature->canonical-sexp
   (signature-field "foo" #:version "2")))

(test-assert "valid narinfo-signature->canonical-sexp"
  (canonical-sexp? (narinfo-signature->canonical-sexp (signature-field "foo"))))


(define %main-substitute-directory
  ;; The place where 'call-with-narinfo' stores its data by default.
  (uri-path (string->uri (getenv "GUIX_BINARY_SUBSTITUTE_URL"))))

(define %alternate-substitute-directory
  ;; Another place.
  (string-append (dirname %main-substitute-directory)
                 "/substituter-alt-data"))

(define %unroutable-substitute-url
  ;; Substitute URL with an unroutable server address, as per
  ;; <https://www.rfc-editor.org/rfc/rfc5737>.
  "http://203.0.113.1")


(define %narinfo
  ;; Skeleton of the narinfo used below.
  (string-append "StorePath: " (%store-prefix)
                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
URL: example.nar
Compression: none
NarHash: sha256:" (bytevector->nix-base32-string
                   (sha256 (string->utf8 "Substitutable data."))) "
NarSize: 42
References: bar baz
Deriver: " (%store-prefix) "/foo.drv
System: mips64el-linux\n"))

(define* (call-with-narinfo narinfo thunk
                            #:optional
                            (narinfo-directory %main-substitute-directory))
  "Call THUNK in a context where the directory at URL is populated with
a file for NARINFO."
  (mkdir-p narinfo-directory)
  (let ((cache-directory (string-append (getenv "XDG_CACHE_HOME")
                                        "/guix/substitute/")))
    (dynamic-wind
      (lambda ()
        (when (file-exists? cache-directory)
          (delete-file-recursively cache-directory))
        (call-with-output-file (string-append narinfo-directory
                                              "/nix-cache-info")
          (lambda (port)
            (format port "StoreDir: ~a\nWantMassQuery: 0\n"
                    (%store-prefix))))
        (call-with-output-file (string-append narinfo-directory "/"
                                              "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
                                              ".narinfo")
          (cut display narinfo <>))

        ;; Prepare the nar.
        (call-with-output-file
            (string-append narinfo-directory "/example.out")
          (cut display "Substitutable data." <>))
        (call-with-output-file
            (string-append narinfo-directory "/example.nar")
          (cute write-file
                (string-append narinfo-directory "/example.out") <>))

        (%allow-unauthenticated-substitutes? #f))
      thunk
      (lambda ()
        (when (file-exists? cache-directory)
          (delete-file-recursively cache-directory))))))

(define-syntax-rule (with-narinfo narinfo body ...)
  (call-with-narinfo narinfo (lambda () body ...)))

(define-syntax-rule (with-narinfo* narinfo directory body ...)
  (call-with-narinfo narinfo (lambda () body ...) directory))

;; Transmit these options to 'guix substitute'.
(substitute-urls (list (getenv "GUIX_BINARY_SUBSTITUTE_URL")))

;; Never use file descriptor 4, unlike what happens when invoked by the
;; daemon.
(%reply-file-descriptor #f)


(test-equal "query narinfo without signature"
  ""                                              ; not substitutable

  (with-narinfo %narinfo
    (string-trim-both
     (with-output-to-string
       (lambda ()
         (with-input-from-string (string-append "have " (%store-prefix)
                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
           (lambda ()
             (guix-substitute "--query"))))))))

(test-equal "query narinfo with invalid hash"
  ;; The hash in the signature differs from the hash of %NARINFO.
  ""

  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field "different body")
                               "\n")
    (string-trim-both
     (with-output-to-string
       (lambda ()
         (with-input-from-string (string-append "have " (%store-prefix)
                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
           (lambda ()
             (guix-substitute "--query"))))))))

(test-equal "query narinfo with signature over nothing"
  ;; The signature is computed over the empty string, not over the important
  ;; parts, so the narinfo must be ignored.
  ""

  (with-narinfo (string-append "Signature: " (signature-field "") "\n"
                                %narinfo "\n")
    (string-trim-both
     (with-output-to-string
       (lambda ()
         (with-input-from-string (string-append "have " (%store-prefix)
                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
           (lambda ()
             (guix-substitute "--query"))))))))

(test-equal "query narinfo with signature over irrelevant bits"
  ;; The signature is valid but it does not cover the
  ;; StorePath/NarHash/References tuple and is thus irrelevant; the narinfo
  ;; must be ignored.
  ""

  (let ((prefix (string-append "StorePath: " (%store-prefix)
                               "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
URL: example.nar
Compression: none\n")))
    (with-narinfo (string-append prefix
                                 "Signature: " (signature-field prefix) "
NarHash: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
NarSize: 42
References: bar baz
Deriver: " (%store-prefix) "/foo.drv
System: mips64el-linux\n")
      (string-trim-both
       (with-output-to-string
         (lambda ()
           (with-input-from-string (string-append "have " (%store-prefix)
                                                  "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
             (lambda ()
               (guix-substitute "--query")))))))))

(test-equal "query narinfo with signature over relevant subset"
  ;; The signature covers the StorePath/NarHash/References tuple, so it is
  ;; valid; it does not cover non-normative fields, which is fine.
  (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")

  (let ((prefix (string-append "StorePath: " (%store-prefix)
                               "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
NarHash: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
References: bar baz\n")))
    (with-narinfo (string-append prefix
                                 "Signature: " (signature-field prefix) "
URL: example.nar
Compression: none
NarSize: 42
Deriver: " (%store-prefix) "/foo.drv")
      (string-trim-both
       (with-output-to-string
         (lambda ()
           (with-input-from-string (string-append "have " (%store-prefix)
                                                  "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
             (lambda ()
               (guix-substitute "--query")))))))))

(test-equal "query narinfo signed with authorized key"
  (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")

  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field %narinfo)
                               "\n")
    (string-trim-both
     (with-output-to-string
       (lambda ()
         (with-input-from-string (string-append "have " (%store-prefix)
                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
           (lambda ()
             (guix-substitute "--query"))))))))

(test-equal "query narinfo signed with authorized key, unroutable URL first"
  (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")

  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field %narinfo)
                               "\n")
    (string-trim-both
     (with-output-to-string
       (lambda ()
         (with-input-from-string (string-append "have " (%store-prefix)
                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
           (lambda ()
             (parameterize ((substitute-urls
                             (list %unroutable-substitute-url
                                   (string-append "file://"
                                                  %main-substitute-directory))))
               (guix-substitute "--query")))))))))

(test-equal "query narinfo signed with unauthorized key"
  ""                                              ; not substitutable

  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field
                                %narinfo
                                #:public-key %wrong-public-key)
                               "\n")
    (string-trim-both
     (with-output-to-string
       (lambda ()
         (with-input-from-string (string-append "have " (%store-prefix)
                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
           (lambda ()
             (guix-substitute "--query"))))))))

(test-quit "substitute, no signature"
    "no valid substitute"
  (with-narinfo %narinfo
    (with-input-from-string (string-append "substitute "
                                           (%store-prefix)
                                           "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
                                           " foo\n")
      (lambda ()
        (guix-substitute "--substitute")))))

(test-quit "substitute, invalid narinfo hash"
    "no valid substitute"
  ;; The hash in the signature differs from the hash of %NARINFO.
  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field "different body")
                               "\n")
    (with-input-from-string (string-append "substitute "
                                           (%store-prefix)
                                           "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
                                           " foo\n")
      (lambda ()
        (guix-substitute "--substitute")))))

(test-equal "substitute, invalid hash"
  (string-append "hash-mismatch sha256 "
                 (bytevector->nix-base32-string (sha256 #vu8())) " "
                 (let-values (((port get-hash)
                               (open-hash-port (hash-algorithm sha256)))
                              ((content)
                               "Substitutable data."))
                   (write-file-tree "foo" port
                                    #:file-type+size
                                    (lambda _
                                      (values 'regular
                                              (string-length content)))
                                    #:file-port
                                    (lambda _
                                      (open-input-string content)))
                   (close-port port)
                   (bytevector->nix-base32-string (get-hash)))
                 "\n")

  ;; Arrange so the actual data hash does not match the 'NarHash' field in the
  ;; narinfo.
  (with-output-to-string
    (lambda ()
      (let ((narinfo (string-append "StorePath: " (%store-prefix)
                                    "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-wrong-hash
URL: example.nar
Compression: none
NarHash: sha256:" (bytevector->nix-base32-string (sha256 #vu8())) "
NarSize: 42
References: 
Deriver: " (%store-prefix) "/foo.drv
System: mips64el-linux\n")))
        (with-narinfo (string-append narinfo "Signature: "
                                     (signature-field narinfo) "\n")
          (call-with-temporary-directory
           (lambda (directory)
             (with-input-from-string (string-append
                                      "substitute " (%store-prefix)
                                      "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-wrong-hash "
                                      directory "/wrong-hash\n")
               (lambda ()
                 (guix-substitute "--substitute"))))))))))

(test-quit "substitute, unauthorized key"
    "no valid substitute"
  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field
                                %narinfo
                                #:public-key %wrong-public-key)
                               "\n")
    (with-input-from-string (string-append "substitute "
                                           (%store-prefix)
                                           "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
                                           " foo\n")
      (lambda ()
        (guix-substitute "--substitute")))))

(test-equal "substitute, authorized key"
  '("Substitutable data." 1 #o444)
  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field %narinfo))
    (dynamic-wind
      (const #t)
      (lambda ()
        (request-substitution (string-append (%store-prefix)
                                             "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                              "substitute-retrieved")
        (list (call-with-input-file "substitute-retrieved" get-string-all)
              (stat:mtime (lstat "substitute-retrieved"))
              (stat:perms (lstat "substitute-retrieved"))))
      (lambda ()
        (false-if-exception (delete-file "substitute-retrieved"))))))

(test-equal "substitute, authorized key, first substitute URL is unroutable"
  '("Substitutable data." 1 #o444)
  (with-narinfo (string-append %narinfo "Signature: "
                               (signature-field %narinfo))
    (dynamic-wind
      (const #t)
      (lambda ()
        ;; Pick an unroutable URL as the first one.  This shouldn't be a
        ;; problem.
        (parameterize ((substitute-urls
                        (list %unroutable-substitute-url
                              (string-append "file://"
                                             %main-substitute-directory))))
          (request-substitution (string-append (%store-prefix)
                                               "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                "substitute-retrieved")
          (list (call-with-input-file "substitute-retrieved" get-string-all)
                (stat:mtime (lstat "substitute-retrieved"))
                (stat:perms (lstat "substitute-retrieved")))))
      (lambda ()
        (false-if-exception (delete-file "substitute-retrieved"))))))

(test-equal "substitute, unauthorized narinfo comes first"
  "Substitutable data."
  (with-narinfo*
      (string-append %narinfo "Signature: "
                     (signature-field
                      %narinfo
                      #:public-key %wrong-public-key))
      %alternate-substitute-directory

    (with-narinfo* (string-append %narinfo "Signature: "
                                  (signature-field %narinfo))
        %main-substitute-directory

      (dynamic-wind
        (const #t)
        (lambda ()
          ;; Remove this file so that the substitute can only be retrieved
          ;; from %ALTERNATE-SUBSTITUTE-DIRECTORY.
          (delete-file (string-append %main-substitute-directory
                                      "/example.nar"))

          (parameterize ((substitute-urls
                          (map (cut string-append "file://" <>)
                               (list %alternate-substitute-directory
                                     %main-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-equal "substitute, unsigned narinfo comes first"
  "Substitutable data."
  (with-narinfo* %narinfo                         ;not signed!
      %alternate-substitute-directory

    (with-narinfo* (string-append %narinfo "Signature: "
                                  (signature-field %narinfo))
        %main-substitute-directory

      (dynamic-wind
        (const #t)
        (lambda ()
          ;; Remove this file so that the substitute can only be retrieved
          ;; from %ALTERNATE-SUBSTITUTE-DIRECTORY.
          (delete-file (string-append %main-substitute-directory
                                      "/example.nar"))

          (parameterize ((substitute-urls
                          (map (cut string-append "file://" <>)
                               (list %alternate-substitute-directory
                                     %main-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-equal "substitute, first URL has narinfo but lacks nar, second URL unauthorized"
  "Substitutable data."
  (with-narinfo*
      (string-append %narinfo "Signature: "
                     (signature-field
                      %narinfo
                      #:public-key %wrong-public-key))
      %alternate-substitute-directory

    (with-narinfo* (string-append %narinfo "Signature: "
                                  (signature-field %narinfo))
        %main-substitute-directory

      (dynamic-wind
        (const #t)
        (lambda ()
          ;; Remove this file so that the substitute can only be retrieved
          ;; from %ALTERNATE-SUBSTITUTE-DIRECTORY.
          (delete-file (string-append %main-substitute-directory
                                      "/example.nar"))

          (parameterize ((substitute-urls
                          (map (cut string-append "file://" <>)
                               (list %main-substitute-directory
                                     %alternate-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-equal "substitute, first URL has narinfo but nar is 404, both URLs authorized"
  "Substitutable data."
  (with-narinfo*
      (string-append %narinfo "Signature: "
                     (signature-field %narinfo))
      %main-substitute-directory

    (with-http-server `((200 ,(string-append %narinfo "Signature: "
                                             (signature-field %narinfo)))
                        (404 "Sorry, nar is missing!"))
      (dynamic-wind
        (const #t)
        (lambda ()
          (parameterize ((substitute-urls
                          (list (%local-url)
                                (string-append "file://"
                                               %main-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-equal "substitute, first URL has narinfo but nar is 404, one URL authorized"
  "Substitutable data."
  (with-narinfo*
      (string-append %narinfo "Signature: "
                     (signature-field
                      %narinfo
                      #:public-key %wrong-public-key))
      %main-substitute-directory

    (with-http-server `((200 ,(string-append %narinfo "Signature: "
                                             (signature-field
                                              %narinfo
                                              #:public-key %wrong-public-key)))
                        (404 "Sorry, nar is missing!"))
      (let ((url1 (%local-url)))
        (parameterize ((%http-server-port 0))
          (with-http-server `((200 ,(string-append %narinfo "Signature: "
                                                   (signature-field %narinfo)))
                              (404 "Sorry, nar is missing!"))
            (let ((url2 (%local-url)))
              (dynamic-wind
                (const #t)
                (lambda ()
                  (parameterize ((substitute-urls
                                  (list url1 url2
                                        (string-append "file://"
                                                       %main-substitute-directory))))
                    (request-substitution (string-append (%store-prefix)
                                                         "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                          "substitute-retrieved"))
                  (call-with-input-file "substitute-retrieved" get-string-all))
                (lambda ()
                  (false-if-exception (delete-file "substitute-retrieved")))))))))))

(test-equal "substitute, preferred nar URL is 404, other is 200"
  "Substitutable data."
  (with-narinfo* (string-append %narinfo "Signature: " (signature-field %narinfo))
      %main-substitute-directory

    (with-http-server `((200 ,(string-append %narinfo "Signature: "
                                             (signature-field %narinfo)
                                             "\n"
                                             "URL: example.nar.lz\n"
                                             "Compression: lzip\n"))
                        (404 "Sorry, nar.lz is missing!")
                        (200 ,(call-with-input-file
                                  (string-append %main-substitute-directory
                                                 "/example.nar")
                                get-bytevector-all)))
      (dynamic-wind
        (const #t)
        (lambda ()
          (parameterize ((substitute-urls (list (%local-url))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-equal "substitute, previous partial download around"
  "Substitutable data."
  (with-narinfo* (string-append %narinfo "Signature: " (signature-field %narinfo))
      %main-substitute-directory

    (with-http-server `((200 ,(string-append %narinfo "Signature: "
                                             (signature-field %narinfo)))
                        (200 ,(call-with-input-file
                                  (string-append %main-substitute-directory
                                                 "/example.nar")
                                get-bytevector-all)))
      (dynamic-wind
        (const #t)
        (lambda ()
          (parameterize ((substitute-urls (list (%local-url))))
            (mkdir-p "substitute-retrieved/a/b/c/d") ;add stale data
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-quit "substitute, narinfo is available but nar is missing"
    "failed to find alternative substitute"
  (with-narinfo*
      (string-append %narinfo "Signature: "
                     (signature-field
                      %narinfo
                      #:public-key %wrong-public-key))
      %main-substitute-directory

    (with-http-server `((200 ,(string-append %narinfo "Signature: "
                                             (signature-field %narinfo)))
                        (404 "Sorry, nar is missing!"))
      (parameterize ((substitute-urls
                      (list (%local-url)
                            (string-append "file://"
                                           %main-substitute-directory))))
        (delete-file (string-append %main-substitute-directory
                                    "/example.nar"))
        (request-substitution (string-append (%store-prefix)
                                             "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                              "substitute-retrieved")
        (not (file-exists? "substitute-retrieved"))))))

(test-equal "substitute, first narinfo is unsigned and has wrong hash"
  "Substitutable data."
  (with-narinfo* (regexp-substitute #f
                                    (string-match "NarHash: [[:graph:]]+"
                                                  %narinfo)
                                    'pre
                                    "NarHash: sha256:"
                                    (bytevector->nix-base32-string
                                     (make-bytevector 32))
                                    'post)
      %alternate-substitute-directory

    (with-narinfo* (string-append %narinfo "Signature: "
                                  (signature-field %narinfo))
        %main-substitute-directory

      (dynamic-wind
        (const #t)
        (lambda ()
          ;; This time remove the file so that the substitute can only be
          ;; retrieved from %MAIN-SUBSTITUTE-DIRECTORY.
          (delete-file (string-append %alternate-substitute-directory
                                      "/example.nar"))

          (parameterize ((substitute-urls
                          (map (cut string-append "file://" <>)
                               (list %alternate-substitute-directory
                                     %main-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-equal "substitute, first narinfo is unsigned and has wrong refs"
  "Substitutable data."
  (with-narinfo* (regexp-substitute #f
                                    (string-match "References: ([^\n]+)\n"
                                                  %narinfo)
                                    'pre "References: " 1
                                    " wrong set of references\n"
                                    'post)
      %alternate-substitute-directory

    (with-narinfo* (string-append %narinfo "Signature: "
                                  (signature-field %narinfo))
        %main-substitute-directory

      (dynamic-wind
        (const #t)
        (lambda ()
          ;; This time remove the file so that the substitute can only be
          ;; retrieved from %MAIN-SUBSTITUTE-DIRECTORY.
          (delete-file (string-append %alternate-substitute-directory
                                      "/example.nar"))

          (parameterize ((substitute-urls
                          (map (cut string-append "file://" <>)
                               (list %alternate-substitute-directory
                                     %main-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-quit "substitute, two invalid narinfos"
    "no valid substitute"
  (with-narinfo* %narinfo                         ;not signed
      %alternate-substitute-directory

    (with-narinfo* (string-append %narinfo "Signature: " ;unauthorized
                                  (signature-field
                                   %narinfo
                                   #:public-key %wrong-public-key))
        %main-substitute-directory

      (with-input-from-string (string-append "substitute "
                                             (%store-prefix)
                                             "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
                                             " substitute-retrieved\n")
        (lambda ()
          (guix-substitute "--substitute"))))))

(test-equal "substitute, narinfo with several URLs"
  "Substitutable data."
  (let ((narinfo (string-append "StorePath: " (%store-prefix)
                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
URL: example.nar.gz
Compression: gzip
URL: example.nar.lz
Compression: lzip
URL: example.nar
Compression: none
NarHash: sha256:" (bytevector->nix-base32-string
                   (sha256 (string->utf8 "Substitutable data."))) "
NarSize: 42
References: bar baz
Deriver: " (%store-prefix) "/foo.drv
System: mips64el-linux\n")))
    (with-narinfo (string-append narinfo "Signature: "
                                 (signature-field narinfo))
      (dynamic-wind
        (const #t)
        (lambda ()
          (define (compress input output compression)
            (call-with-output-file output
              (lambda (port)
                (call-with-compressed-output-port compression port
                  (lambda (port)
                    (call-with-input-file input
                      (lambda (input)
                        (dump-port input port))))))))

          (let ((nar (string-append %main-substitute-directory
                                    "/example.nar")))
            (compress nar (string-append nar ".gz") 'gzip)
            (compress nar (string-append nar ".lz") 'lzip))

          (parameterize ((substitute-urls
                          (list (string-append "file://"
                                               %main-substitute-directory))))
            (request-substitution (string-append (%store-prefix)
                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
                                  "substitute-retrieved"))
          (call-with-input-file "substitute-retrieved" get-string-all))
        (lambda ()
          (false-if-exception (delete-file "substitute-retrieved")))))))

(test-end "substitute")

;;; Local Variables:
;;; eval: (put 'with-narinfo 'scheme-indent-function 1)
;;; eval: (put 'with-narinfo* 'scheme-indent-function 2)
;;; eval: (put 'test-quit 'scheme-indent-function 2)
;;; End:
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								;;; GNU Guix --- Functional package management for GNU
 								;;; Copyright © 2014 Nikita Karetnikov <nikita@karetnikov.org>
-												substitute: Gracefully retry after failed partial downloads.

Fixes <https://issues.guix.gnu.org/63443>.
Reported by Attila Lendvai <attila@lendvai.name>.

* guix/scripts/substitute.scm (catch-system-error): New macro.
(download-nar): Add call to 'delete-file-recursively'.
* tests/substitute.scm ("substitute, previous partial download around"):
New test.

											
										
										
											2023-06-08 20:43:05 +00:00
+								;;; Copyright © 2014-2015, 2017-2019, 2021-2023 Ludovic Courtès <ludo@gnu.org>
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								;;;
 								;;; This file is part of GNU Guix.
 								;;;
 								;;; GNU Guix is free software; you can redistribute it and/or modify it
 								;;; under the terms of the GNU General Public License as published by
 								;;; the Free Software Foundation; either version 3 of the License, or (at
 								;;; your option) any later version.
 								;;;
 								;;; GNU Guix is distributed in the hope that it will be useful, but
 								;;; WITHOUT ANY WARRANTY; without even the implied warranty of
 								;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 								;;; GNU General Public License for more details.
 								;;;
 								;;; You should have received a copy of the GNU General Public License
 								;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								(define-module (test-substitute)
 								  #:use-module (guix scripts substitute)
-												guix: Move narinfo code from substitute script to module.

This separation between the code for dealing with narinfos from the code doing
that for a purpose should make things clearer, and better support components
other that the substitute script in using this code.

This is just moving the code around, no code should have been significantly
changed.

* guix/scripts/substitute.scm (<narinfo>): Move record type to (guix narinfo).
(fields->alist, narinfo-hash-algorithm+value, narinfo-hash->sha256,
narinfo-signature->canonical-sexp, narinfo-maker, read-narinfo,
narinfo-sha256, valid-narinfo?, write-narinfo, narinfo->string,
string->narinfo, equivalent-narinfo?, supported-compression?,
compresses-better?, narinfo-best-uri): Move procedures to (guix narinfo).
(%compression-methods): Move variable to (guix narinfo).
* guix/narinfo.scm: New file.
* Makefile.am (MODULES): Add it.
* po/guix/POTFILES.in: Add 'guix/narinfo.scm'.

											
										
										
											2020-12-24 15:50:54 +00:00
+								  #:use-module (guix narinfo)
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								  #:use-module (guix base64)
-												Switch to Guile-Gcrypt.

This removes (guix hash) and (guix pk-crypto), which now live as part of
Guile-Gcrypt (version 0.1.0.)

* guix/gcrypt.scm, guix/hash.scm, guix/pk-crypto.scm,
tests/hash.scm, tests/pk-crypto.scm: Remove.
* configure.ac: Test for Guile-Gcrypt.  Remove LIBGCRYPT and
LIBGCRYPT_LIBDIR assignments.
* m4/guix.m4 (GUIX_ASSERT_LIBGCRYPT_USABLE): Remove.
* README: Add Guile-Gcrypt to the dependencies; move libgcrypt as
"required unless --disable-daemon".
* doc/guix.texi (Requirements): Likewise.
* gnu/packages/bash.scm, guix/derivations.scm, guix/docker.scm,
guix/git.scm, guix/http-client.scm, guix/import/cpan.scm,
guix/import/cran.scm, guix/import/crate.scm, guix/import/elpa.scm,
guix/import/gnu.scm, guix/import/hackage.scm,
guix/import/texlive.scm, guix/import/utils.scm, guix/nar.scm,
guix/pki.scm, guix/scripts/archive.scm,
guix/scripts/authenticate.scm, guix/scripts/download.scm,
guix/scripts/hash.scm, guix/scripts/pack.scm,
guix/scripts/publish.scm, guix/scripts/refresh.scm,
guix/scripts/substitute.scm, guix/store.scm,
guix/store/deduplication.scm, guix/tests.scm, tests/base32.scm,
tests/builders.scm, tests/challenge.scm, tests/cpan.scm,
tests/crate.scm, tests/derivations.scm, tests/gem.scm,
tests/nar.scm, tests/opam.scm, tests/pki.scm,
tests/publish.scm, tests/pypi.scm, tests/store-deduplication.scm,
tests/store.scm, tests/substitute.scm: Adjust imports.
* gnu/system/vm.scm: Likewise.
(guile-sqlite3&co): Rename to...
(gcrypt-sqlite3&co): ... this.  Add GUILE-GCRYPT.
(expression->derivation-in-linux-vm)[config]: Remove.
(iso9660-image)[config]: Remove.
(qemu-image)[config]: Remove.
(system-docker-image)[config]: Remove.
* guix/scripts/pack.scm: Adjust imports.
(guile-sqlite3&co): Rename to...
(gcrypt-sqlite3&co): ... this.  Add GUILE-GCRYPT.
(self-contained-tarball)[build]: Call 'make-config.scm' without
 #:libgcrypt argument.
(squashfs-image)[libgcrypt]: Remove.
[build]: Call 'make-config.scm' without #:libgcrypt.
(docker-image)[config, json]: Remove.
[build]: Add GUILE-GCRYPT to the extensions  Remove (guix config) from
the imported modules.
* guix/self.scm (specification->package): Remove "libgcrypt", add
"guile-gcrypt".
(compiled-guix): Remove #:libgcrypt.
[guile-gcrypt]: New variable.
[dependencies]: Add it.
[*core-modules*]: Remove #:libgcrypt from 'make-config.scm' call.
Add #:extensions.
[*config*]: Remove #:libgcrypt from 'make-config.scm' call.
(%dependency-variables): Remove %libgcrypt.
(make-config.scm): Remove #:libgcrypt.
* build-aux/build-self.scm (guile-gcrypt): New variable.
(make-config.scm): Remove #:libgcrypt.
(build-program)[fake-gcrypt-hash]: New variable.
Add (gcrypt hash) to the imported modules.  Adjust load path
assignments.
* gnu/packages/package-management.scm (guix)[propagated-inputs]: Add
GUILE-GCRYPT.
[arguments]: In 'wrap-program' phase, add GUILE-GCRYPT to the search
path.

											
										
										
											2018-08-31 15:07:07 +00:00
+								  #:use-module (gcrypt hash)
-												Break module cycle involving (guix store) and (guix ui).

Before, there was a cycle along the lines of:

  (guix store) -> (guix nar) -> (guix ui) -> (guix store)

This caused problems, as discussed at:

  http://lists.gnu.org/archive/html/guix-devel/2014-10/msg00109.html

This patch removes cycles in the (guix ...) modules.

* guix/nar.scm (&nar-error, &nar-read-error, dump, write-contents,
  read-contents, %archive-version-1, write-file, restore-file): Move to...
* guix/serialization.scm: ... here.
* guix/store.scm: Remove dependency on (guix nar).
* guix/scripts/hash.scm, guix/scripts/offload.scm,
  guix/scripts/substitute-binary.scm, tests/nar.scm, tests/store.scm,
  tests/substitute-binary.scm: Adjust accordingly.

											
										
										
											2014-10-09 21:46:13 +00:00
+								  #:use-module (guix serialization)
-												Switch to Guile-Gcrypt.

This removes (guix hash) and (guix pk-crypto), which now live as part of
Guile-Gcrypt (version 0.1.0.)

* guix/gcrypt.scm, guix/hash.scm, guix/pk-crypto.scm,
tests/hash.scm, tests/pk-crypto.scm: Remove.
* configure.ac: Test for Guile-Gcrypt.  Remove LIBGCRYPT and
LIBGCRYPT_LIBDIR assignments.
* m4/guix.m4 (GUIX_ASSERT_LIBGCRYPT_USABLE): Remove.
* README: Add Guile-Gcrypt to the dependencies; move libgcrypt as
"required unless --disable-daemon".
* doc/guix.texi (Requirements): Likewise.
* gnu/packages/bash.scm, guix/derivations.scm, guix/docker.scm,
guix/git.scm, guix/http-client.scm, guix/import/cpan.scm,
guix/import/cran.scm, guix/import/crate.scm, guix/import/elpa.scm,
guix/import/gnu.scm, guix/import/hackage.scm,
guix/import/texlive.scm, guix/import/utils.scm, guix/nar.scm,
guix/pki.scm, guix/scripts/archive.scm,
guix/scripts/authenticate.scm, guix/scripts/download.scm,
guix/scripts/hash.scm, guix/scripts/pack.scm,
guix/scripts/publish.scm, guix/scripts/refresh.scm,
guix/scripts/substitute.scm, guix/store.scm,
guix/store/deduplication.scm, guix/tests.scm, tests/base32.scm,
tests/builders.scm, tests/challenge.scm, tests/cpan.scm,
tests/crate.scm, tests/derivations.scm, tests/gem.scm,
tests/nar.scm, tests/opam.scm, tests/pki.scm,
tests/publish.scm, tests/pypi.scm, tests/store-deduplication.scm,
tests/store.scm, tests/substitute.scm: Adjust imports.
* gnu/system/vm.scm: Likewise.
(guile-sqlite3&co): Rename to...
(gcrypt-sqlite3&co): ... this.  Add GUILE-GCRYPT.
(expression->derivation-in-linux-vm)[config]: Remove.
(iso9660-image)[config]: Remove.
(qemu-image)[config]: Remove.
(system-docker-image)[config]: Remove.
* guix/scripts/pack.scm: Adjust imports.
(guile-sqlite3&co): Rename to...
(gcrypt-sqlite3&co): ... this.  Add GUILE-GCRYPT.
(self-contained-tarball)[build]: Call 'make-config.scm' without
 #:libgcrypt argument.
(squashfs-image)[libgcrypt]: Remove.
[build]: Call 'make-config.scm' without #:libgcrypt.
(docker-image)[config, json]: Remove.
[build]: Add GUILE-GCRYPT to the extensions  Remove (guix config) from
the imported modules.
* guix/self.scm (specification->package): Remove "libgcrypt", add
"guile-gcrypt".
(compiled-guix): Remove #:libgcrypt.
[guile-gcrypt]: New variable.
[dependencies]: Add it.
[*core-modules*]: Remove #:libgcrypt from 'make-config.scm' call.
Add #:extensions.
[*config*]: Remove #:libgcrypt from 'make-config.scm' call.
(%dependency-variables): Remove %libgcrypt.
(make-config.scm): Remove #:libgcrypt.
* build-aux/build-self.scm (guile-gcrypt): New variable.
(make-config.scm): Remove #:libgcrypt.
(build-program)[fake-gcrypt-hash]: New variable.
Add (gcrypt hash) to the imported modules.  Adjust load path
assignments.
* gnu/packages/package-management.scm (guix)[propagated-inputs]: Add
GUILE-GCRYPT.
[arguments]: In 'wrap-program' phase, add GUILE-GCRYPT to the search
path.

											
										
										
											2018-08-31 15:07:07 +00:00
+								  #:use-module (gcrypt pk-crypto)
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								  #:use-module (guix pki)
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  #:use-module (guix config)
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								  #:use-module (guix base32)
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  #:use-module ((guix store) #:select (%store-prefix))
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								  #:use-module ((guix ui) #:select (guix-warning-port))
-												daemon: Let 'guix substitute' perform hash checks.

This way, the hash of the store item can be computed as it is restored,
thereby avoiding an additional file tree traversal ('hashPath' call)
later on in the daemon.  Consequently, it should reduce latency between
subsequent substitute downloads.

This is a followup to 5ff521452b9ec2aae9ed8e4bb7bdc250a581f203.

* guix/scripts/substitute.scm (narinfo-hash-algorithm+value): New
procedure.
(process-substitution): Wrap INPUT into a hash input port, 'hashed', and
read from it.  Compare the actual and expected hashes, and print a
"hash-mismatch" status line when they differ.  When they match, print
not just "success" but also the nar hash and size.
* nix/libstore/build.cc (class SubstitutionGoal)[expectedHashStr]:
Remove.
(SubstitutionGoal::finished): Tokenize 'status'.  Parse it and handle
"success" and "hash-mismatch" accordingly.  Call 'hashPath' only when
the returned hash is not SHA256.
(SubstitutionGoal::handleChildOutput): Remove 'expectedHashStr'
handling.
* tests/substitute.scm ("substitute, invalid hash"): Rename to...
("substitute, invalid narinfo hash"): ... this.
("substitute, invalid hash"): New test.

											
										
										
											2020-12-13 21:46:03 +00:00
+								  #:use-module ((guix utils)
 								                #:select (call-with-temporary-directory
 								                          call-with-compressed-output-port))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								  #:use-module ((guix build utils)
-												substitute: Select the best compression methods.

When a server publishes several URLs with different compression methods,
'guix substitute' can now choose the best one among the compression
methods that it supports.

* guix/scripts/substitute.scm (<narinfo>)[uri]: Replace with...
[uris]: ... this.
[compression]: Replace with...
[compressions]: ... this.
[file-size]: Replace with...
[file-sizes]: ... this.
[file-hash]: Replace with...
[file-hashes]: ... this.
(narinfo-maker): Adjust accordingly.  Ensure 'file-sizes' and
'file-hashes' have the right length.
(assert-valid-signature, valid-narinfo?): Use the first element of
'narinfo-uris' in error messages.
(read-narinfo): Expect "URL", "Compression", "FileSize", and "FileHash"
to occur multiple times.
(display-narinfo-data): Call 'select-uri' to determine the file size.
(%compression-methods): New variable.
(supported-compression?, compresses-better?, select-uri): New
procedures.
(process-substitution): Call 'select-uri' to select the URI and
compression.
* guix/scripts/weather.scm (report-server-coverage): Account for all the
values returned by 'narinfo-file-sizes'.
* tests/substitute.scm ("substitute, narinfo with several URLs"): New
test.

											
										
										
											2019-05-31 14:26:08 +00:00
+								                #:select (mkdir-p delete-file-recursively dump-port))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								  #:use-module (guix tests http)
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								  #:use-module (rnrs bytevectors)
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  #:use-module (rnrs io ports)
 								  #:use-module (web uri)
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  #:use-module (ice-9 regex)
-												daemon: Let 'guix substitute' perform hash checks.

This way, the hash of the store item can be computed as it is restored,
thereby avoiding an additional file tree traversal ('hashPath' call)
later on in the daemon.  Consequently, it should reduce latency between
subsequent substitute downloads.

This is a followup to 5ff521452b9ec2aae9ed8e4bb7bdc250a581f203.

* guix/scripts/substitute.scm (narinfo-hash-algorithm+value): New
procedure.
(process-substitution): Wrap INPUT into a hash input port, 'hashed', and
read from it.  Compare the actual and expected hashes, and print a
"hash-mismatch" status line when they differ.  When they match, print
not just "success" but also the nar hash and size.
* nix/libstore/build.cc (class SubstitutionGoal)[expectedHashStr]:
Remove.
(SubstitutionGoal::finished): Tokenize 'status'.  Parse it and handle
"success" and "hash-mismatch" accordingly.  Call 'hashPath' only when
the returned hash is not SHA256.
(SubstitutionGoal::handleChildOutput): Remove 'expectedHashStr'
handling.
* tests/substitute.scm ("substitute, invalid hash"): Rename to...
("substitute, invalid narinfo hash"): ... this.
("substitute, invalid hash"): New test.

											
										
										
											2020-12-13 21:46:03 +00:00
+								  #:use-module (srfi srfi-11)
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  #:use-module (srfi srfi-26)
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								  #:use-module (srfi srfi-34)
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  #:use-module (srfi srfi-35)
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								  #:use-module ((srfi srfi-64) #:hide (test-error)))
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								(define-syntax-rule (test-quit name error-rx exp)
 								  "Emit a test that passes when EXP throws to 'quit' with value 1, and when
 								it writes to GUIX-WARNING-PORT a messages that matches ERROR-RX."
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  (test-equal name
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								    '(1 #t)
 								    (let ((error-output (open-output-string)))
-												daemon: Use 'Agent' to spawn 'guix substitute --query'.

* nix/libstore/local-store.hh (RunningSubstituter): Remove.
(LocalStore)[runningSubstituter]: Change to unique_ptr<Agent>.
[setSubstituterEnv, didSetSubstituterEnv]: Remove.
[getLineFromSubstituter, getIntLineFromSubstituter]: Take an 'Agent'.
* nix/libstore/local-store.cc (LocalStore::~LocalStore): Remove
reference to 'runningSubstituter'.
(LocalStore::setSubstituterEnv, LocalStore::startSubstituter): Remove.
(LocalStore::getLineFromSubstituter): Adjust to 'run' being an 'Agent'.
(LocalStore::querySubstitutablePaths): Spawn substituter agent if
needed.  Adjust to 'Agent' interface.
(LocalStore::querySubstitutablePathInfos): Likewise.
* nix/libstore/build.cc (SubstitutionGoal::tryToRun): Remove call to
'setSubstituterEnv' and add 'setenv' call for "_NIX_OPTIONS" instead.
(SubstitutionGoal::finished): Remove 'readLine' call for 'dummy'.
* guix/scripts/substitute.scm (%allow-unauthenticated-substitutes?):
Remove second argument to 'make-parameter'.
(process-query): Call 'warn-about-missing-authentication'
when (%allow-unauthenticated-substitutes?) is #t.
(guix-substitute): Wrap body in 'parameterize'.  Set 'guix-warning-port'
too.  No longer exit when 'substitute-urls' returns the empty list.  No
longer print newline initially.
* tests/substitute.scm (test-quit): Parameterize 'current-error-port' to
account for the port changes in 'guix-substitute'.

											
										
										
											2020-12-01 14:01:40 +00:00
+								      (parameterize ((current-error-port error-output)
 								                     (guix-warning-port error-output))
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								        (catch 'quit
 								          (lambda ()
 								            exp
 								            #f)
 								          (lambda (key value)
 								            (list value
 								                  (let ((message (get-output-string error-output)))
 								                    (->bool (string-match error-rx message))))))))))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								(define (request-substitution item destination)
 								  "Run 'guix substitute --substitute' to fetch ITEM to DESTINATION."
-												substitute: If a server's nar URL is 404, try the next one(s).

If a substitute server advertises in its narinfo, for example, both a
/zstd and a /lzip URL but the /zstd URL is unreachable, try the /lzip
URL.

Fixes <https://issues.guix.gnu.org/63634>.

* guix/narinfo.scm (narinfo-preferred-uris): New procedure.
(narinfo-best-uri): Rebase on top of it.
* guix/scripts/substitute.scm (download-nar)[try-fetch]: New procedure.
Use 'narinfo-preferred-uris' and 'try-fetch' to attempt all the URLs of
NARINFO.
* tests/substitute.scm (request-substitution): Remove 'parameterize'.
Delete DESTINATION.
("substitute, preferred nar URL is 404, other is 200"): New test.

											
										
										
											2023-05-22 15:19:39 +00:00
+								  (false-if-exception (delete-file destination))
 								  (with-input-from-string (string-append "substitute " item " "
 								                                         destination "\n")
 								    (lambda ()
 								      (guix-substitute "--substitute"))))
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								(define %public-key
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  ;; This key is known to be in the ACL by default.
 								  (call-with-input-file (string-append %config-directory "/signing-key.pub")
 								    (compose string->canonical-sexp get-string-all)))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
 								(define %private-key
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  (call-with-input-file (string-append %config-directory "/signing-key.sec")
 								    (compose string->canonical-sexp get-string-all)))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								(define* (signature-body bv #:key (public-key %public-key))
 								  "Return the signature of BV as the base64-encoded body of a narinfo's
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								'Signature' field."
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								  (base64-encode
 								   (string->utf8
 								    (canonical-sexp->string
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								     (signature-sexp (bytevector->hash-data (sha256 bv)
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								                                            #:key-type 'rsa)
 								                     %private-key
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								                     public-key)))))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
 								(define %wrong-public-key
 								  (string->canonical-sexp "(public-key
 								 (rsa
 								  (n #00E05873AC2B168760343145918E954EE9AB73C026355693B192E01EE835261AA689E9EF46642E895BCD65C648524059FC450E4BA77A68F4C52D0E39EF0CC9359709AB6AAB153B63782201871325B0FDA19CB401CD99FD0C31A91CA9000AA90A77E82B89E036FB63BC1D3961207469B3B12468977148D376F8012BB12A4B11A8F1#)
 								  (e #010001#)
 								  )
 								 )"))
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								(define* (signature-field bv-or-str
 								                          #:key (version "1") (public-key %public-key))
 								  "Return the 'Signature' field value of bytevector/string BV-OR-STR, using
 								PUBLIC-KEY as the signature's principal, and using VERSION as the signature
 								version identifier.."
 								  (string-append version ";example.gnu.org;"
 								                 (signature-body (if (string? bv-or-str)
 								                                     (string->utf8 bv-or-str)
 								                                     bv-or-str)
 								                                 #:public-key public-key)))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								(test-begin "substitute")
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								(test-quit "not a number"
 								    "signature version"
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (narinfo-signature->canonical-sexp
 								   (signature-field "foo" #:version "not a number")))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								(test-quit "wrong version number"
 								    "unsupported.*version"
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (narinfo-signature->canonical-sexp
 								   (signature-field "foo" #:version "2")))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
 								(test-assert "valid narinfo-signature->canonical-sexp"
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (canonical-sexp? (narinfo-signature->canonical-sexp (signature-field "foo"))))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								(define %main-substitute-directory
 								  ;; The place where 'call-with-narinfo' stores its data by default.
 								  (uri-path (string->uri (getenv "GUIX_BINARY_SUBSTITUTE_URL"))))
 								(define %alternate-substitute-directory
 								  ;; Another place.
 								  (string-append (dirname %main-substitute-directory)
 								                 "/substituter-alt-data"))
-												substitute: Test behavior with unroutable substitute server addresses.

* tests/substitute.scm (%unroutable-substitute-url): New variable.
("query narinfo signed with authorized key, unroutable URL first")
("substitute, authorized key, first substitute URL is unroutable"): New
tests.

											
										
										
											2022-09-21 15:21:48 +00:00
+								(define %unroutable-substitute-url
 								  ;; Substitute URL with an unroutable server address, as per
 								  ;; <https://www.rfc-editor.org/rfc/rfc5737>.
 								  "http://203.0.113.1")
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								(define %narinfo
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  ;; Skeleton of the narinfo used below.
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  (string-append "StorePath: " (%store-prefix)
 								                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								URL: example.nar
 								Compression: none
 								NarHash: sha256:" (bytevector->nix-base32-string
 								                   (sha256 (string->utf8 "Substitutable data."))) "
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
+								NarSize: 42
 								References: bar baz
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								Deriver: " (%store-prefix) "/foo.drv
 								System: mips64el-linux\n"))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								(define* (call-with-narinfo narinfo thunk
 								                            #:optional
 								                            (narinfo-directory %main-substitute-directory))
 								  "Call THUNK in a context where the directory at URL is populated with
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								a file for NARINFO."
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								  (mkdir-p narinfo-directory)
 								  (let ((cache-directory (string-append (getenv "XDG_CACHE_HOME")
 								                                        "/guix/substitute/")))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								    (dynamic-wind
 								      (lambda ()
 								        (when (file-exists? cache-directory)
 								          (delete-file-recursively cache-directory))
 								        (call-with-output-file (string-append narinfo-directory
 								                                              "/nix-cache-info")
 								          (lambda (port)
 								            (format port "StoreDir: ~a\nWantMassQuery: 0\n"
 								                    (%store-prefix))))
 								        (call-with-output-file (string-append narinfo-directory "/"
 								                                              "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 								                                              ".narinfo")
 								          (cut display narinfo <>))
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								        ;; Prepare the nar.
 								        (call-with-output-file
 								            (string-append narinfo-directory "/example.out")
 								          (cut display "Substitutable data." <>))
 								        (call-with-output-file
 								            (string-append narinfo-directory "/example.nar")
 								          (cute write-file
 								                (string-append narinfo-directory "/example.out") <>))
-												substitute: Make '%allow-unauthenticated-substitutes?' public.

* guix/scripts/substitute.scm (warn-about-missing-authentication): New
procedure.
(%allow-unauthenticated-substitutes?): Turn into a public parameter and
use 'warn-about-missing-authentication'.
(valid-narinfo?): Adjust accordingly.
* tests/substitute.scm (call-with-narinfo): Likewise.

											
										
										
											2019-11-26 11:30:45 +00:00
+								        (%allow-unauthenticated-substitutes? #f))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								      thunk
 								      (lambda ()
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								        (when (file-exists? cache-directory)
 								          (delete-file-recursively cache-directory))))))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
 								(define-syntax-rule (with-narinfo narinfo body ...)
 								  (call-with-narinfo narinfo (lambda () body ...)))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								(define-syntax-rule (with-narinfo* narinfo directory body ...)
 								  (call-with-narinfo narinfo (lambda () body ...) directory))
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								;; Transmit these options to 'guix substitute'.
-												substitute: Make substitute URLs a SRFI-39 parameter.

* guix/scripts/substitute.scm (%cache-urls): Rename to...
(%default-substitute-urls): ... this.
(substitute-urls): New variable.
(guix-substitute): Use it instead of %CACHE-URLS.
* tests/substitute.scm: Likewise.

											
										
										
											2017-08-31 21:27:26 +00:00
+								(substitute-urls (list (getenv "GUIX_BINARY_SUBSTITUTE_URL")))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								;; Never use file descriptor 4, unlike what happens when invoked by the
 								;; daemon.
-												daemon: 'guix substitute' replies on FD 4.

This avoids the situation where error messages would unintentionally go
to stderr and be wrongfully interpreted as a reply by the daemon.

Fixes <https://bugs.gnu.org/46362>.
This is a followup to ee3226e9d54891c7e696912245e4904435be191c.

* guix/scripts/substitute.scm (display-narinfo-data): Add 'port'
parameter and honor it.
(process-query): Likewise.
(process-substitution): Likewise.
(%error-to-file-descriptor-4?, with-redirected-error-port): Remove.
(%reply-file-descriptor): New variable.
(guix-substitute): Remove use of 'with-redirected-error-port'.  Define
'reply-port' and pass it to 'process-query' and 'process-substitution'.
* nix/libstore/build.cc (SubstitutionGoal::handleChildOutput): Swap
'builderOut' and 'fromAgent'.
* nix/libstore/local-store.cc (LocalStore::getLineFromSubstituter):
Likewise.
* tests/substitute.scm <top level>: Set '%reply-file-descriptor'
rather than '%error-to-file-descriptor-4?'.

											
										
										
											2021-04-06 10:10:29 +00:00
+								(%reply-file-descriptor #f)
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								(test-equal "query narinfo without signature"
 								  ""                                              ; not substitutable
 								  (with-narinfo %narinfo
 								    (string-trim-both
 								     (with-output-to-string
 								       (lambda ()
 								         (with-input-from-string (string-append "have " (%store-prefix)
 								                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								           (lambda ()
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								             (guix-substitute "--query"))))))))
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								(test-equal "query narinfo with invalid hash"
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  ;; The hash in the signature differs from the hash of %NARINFO.
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								  ""
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field "different body")
 								                               "\n")
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								    (string-trim-both
 								     (with-output-to-string
 								       (lambda ()
 								         (with-input-from-string (string-append "have " (%store-prefix)
 								                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								           (lambda ()
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								             (guix-substitute "--query"))))))))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												substitute: Ignore irrelevant narinfo signatures.

Fixes <https://bugs.gnu.org/33733>.

Fixes a bug whereby 'guix substitute' would accept narinfos whose
signature does not cover the StorePath/NarHash/References tuple.

* guix/scripts/substitute.scm (narinfo-sha256)[%mandatory-fields]: New
variable.
Compute SIGNED-FIELDS; return #f unless each of the %MANDATORY-FIELDS
is among SIGNED-FIELDS.
 * tests/substitute.scm ("query narinfo with signature over nothing")
("query narinfo with signature over irrelevant bits"): New tests.

											
										
										
											2018-12-13 18:45:47 +00:00
+								(test-equal "query narinfo with signature over nothing"
 								  ;; The signature is computed over the empty string, not over the important
 								  ;; parts, so the narinfo must be ignored.
 								  ""
 								  (with-narinfo (string-append "Signature: " (signature-field "") "\n"
 								                                %narinfo "\n")
 								    (string-trim-both
 								     (with-output-to-string
 								       (lambda ()
 								         (with-input-from-string (string-append "have " (%store-prefix)
 								                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								           (lambda ()
 								             (guix-substitute "--query"))))))))
 								(test-equal "query narinfo with signature over irrelevant bits"
 								  ;; The signature is valid but it does not cover the
 								  ;; StorePath/NarHash/References tuple and is thus irrelevant; the narinfo
 								  ;; must be ignored.
 								  ""
 								  (let ((prefix (string-append "StorePath: " (%store-prefix)
 								                               "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
 								URL: example.nar
 								Compression: none\n")))
 								    (with-narinfo (string-append prefix
 								                                 "Signature: " (signature-field prefix) "
 								NarHash: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 								NarSize: 42
 								References: bar baz
 								Deriver: " (%store-prefix) "/foo.drv
 								System: mips64el-linux\n")
 								      (string-trim-both
 								       (with-output-to-string
 								         (lambda ()
 								           (with-input-from-string (string-append "have " (%store-prefix)
 								                                                  "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								             (lambda ()
 								               (guix-substitute "--query")))))))))
-												publish: Do not sign the URL/Compression/FileSize narinfo fields.

This will allow mirror operators to alter these non-normative bits of a
narinfo without having to resign narinfos.

* guix/scripts/publish.scm (narinfo-string): Remove
URL/Compression/FileSize from BASE-INFO.  Move them after "Signature".
* tests/publish.scm ("/*.narinfo")
("/*.narinfo with properly encoded '+' sign")
("/*.narinfo with lzip + gzip")
("with cache, lzip + gzip"): Adjust accordingly.
* tests/substitute.scm ("query narinfo with signature over relevant subset"):
New test.

											
										
										
											2022-02-09 17:36:35 +00:00
 								(test-equal "query narinfo with signature over relevant subset"
 								  ;; The signature covers the StorePath/NarHash/References tuple, so it is
 								  ;; valid; it does not cover non-normative fields, which is fine.
 								  (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								  (let ((prefix (string-append "StorePath: " (%store-prefix)
 								                               "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
 								NarHash: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 								References: bar baz\n")))
 								    (with-narinfo (string-append prefix
 								                                 "Signature: " (signature-field prefix) "
 								URL: example.nar
 								Compression: none
 								NarSize: 42
 								Deriver: " (%store-prefix) "/foo.drv")
 								      (string-trim-both
 								       (with-output-to-string
 								         (lambda ()
 								           (with-input-from-string (string-append "have " (%store-prefix)
 								                                                  "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								             (lambda ()
 								               (guix-substitute "--query")))))))))
-												substitute: Ignore irrelevant narinfo signatures.

Fixes <https://bugs.gnu.org/33733>.

Fixes a bug whereby 'guix substitute' would accept narinfos whose
signature does not cover the StorePath/NarHash/References tuple.

* guix/scripts/substitute.scm (narinfo-sha256)[%mandatory-fields]: New
variable.
Compute SIGNED-FIELDS; return #f unless each of the %MANDATORY-FIELDS
is among SIGNED-FIELDS.
 * tests/substitute.scm ("query narinfo with signature over nothing")
("query narinfo with signature over irrelevant bits"): New tests.

											
										
										
											2018-12-13 18:45:47 +00:00
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								(test-equal "query narinfo signed with authorized key"
 								  (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field %narinfo)
 								                               "\n")
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								    (string-trim-both
 								     (with-output-to-string
 								       (lambda ()
 								         (with-input-from-string (string-append "have " (%store-prefix)
 								                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								           (lambda ()
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								             (guix-substitute "--query"))))))))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												substitute: Test behavior with unroutable substitute server addresses.

* tests/substitute.scm (%unroutable-substitute-url): New variable.
("query narinfo signed with authorized key, unroutable URL first")
("substitute, authorized key, first substitute URL is unroutable"): New
tests.

											
										
										
											2022-09-21 15:21:48 +00:00
+								(test-equal "query narinfo signed with authorized key, unroutable URL first"
 								  (string-append (%store-prefix) "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field %narinfo)
 								                               "\n")
 								    (string-trim-both
 								     (with-output-to-string
 								       (lambda ()
 								         (with-input-from-string (string-append "have " (%store-prefix)
 								                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								           (lambda ()
 								             (parameterize ((substitute-urls
 								                             (list %unroutable-substitute-url
 								                                   (string-append "file://"
 								                                                  %main-substitute-directory))))
 								               (guix-substitute "--query")))))))))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								(test-equal "query narinfo signed with unauthorized key"
 								  ""                                              ; not substitutable
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field
 								                                %narinfo
 								                                #:public-key %wrong-public-key)
 								                               "\n")
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								    (string-trim-both
 								     (with-output-to-string
 								       (lambda ()
 								         (with-input-from-string (string-append "have " (%store-prefix)
 								                                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								           (lambda ()
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								             (guix-substitute "--query"))))))))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								(test-quit "substitute, no signature"
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								    "no valid substitute"
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								  (with-narinfo %narinfo
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								    (with-input-from-string (string-append "substitute "
 								                                           (%store-prefix)
 								                                           "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
 								                                           " foo\n")
 								      (lambda ()
 								        (guix-substitute "--substitute")))))
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
-												daemon: Let 'guix substitute' perform hash checks.

This way, the hash of the store item can be computed as it is restored,
thereby avoiding an additional file tree traversal ('hashPath' call)
later on in the daemon.  Consequently, it should reduce latency between
subsequent substitute downloads.

This is a followup to 5ff521452b9ec2aae9ed8e4bb7bdc250a581f203.

* guix/scripts/substitute.scm (narinfo-hash-algorithm+value): New
procedure.
(process-substitution): Wrap INPUT into a hash input port, 'hashed', and
read from it.  Compare the actual and expected hashes, and print a
"hash-mismatch" status line when they differ.  When they match, print
not just "success" but also the nar hash and size.
* nix/libstore/build.cc (class SubstitutionGoal)[expectedHashStr]:
Remove.
(SubstitutionGoal::finished): Tokenize 'status'.  Parse it and handle
"success" and "hash-mismatch" accordingly.  Call 'hashPath' only when
the returned hash is not SHA256.
(SubstitutionGoal::handleChildOutput): Remove 'expectedHashStr'
handling.
* tests/substitute.scm ("substitute, invalid hash"): Rename to...
("substitute, invalid narinfo hash"): ... this.
("substitute, invalid hash"): New test.

											
										
										
											2020-12-13 21:46:03 +00:00
+								(test-quit "substitute, invalid narinfo hash"
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								    "no valid substitute"
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  ;; The hash in the signature differs from the hash of %NARINFO.
 								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field "different body")
 								                               "\n")
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								    (with-input-from-string (string-append "substitute "
 								                                           (%store-prefix)
 								                                           "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
 								                                           " foo\n")
 								      (lambda ()
 								        (guix-substitute "--substitute")))))
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
-												daemon: Let 'guix substitute' perform hash checks.

This way, the hash of the store item can be computed as it is restored,
thereby avoiding an additional file tree traversal ('hashPath' call)
later on in the daemon.  Consequently, it should reduce latency between
subsequent substitute downloads.

This is a followup to 5ff521452b9ec2aae9ed8e4bb7bdc250a581f203.

* guix/scripts/substitute.scm (narinfo-hash-algorithm+value): New
procedure.
(process-substitution): Wrap INPUT into a hash input port, 'hashed', and
read from it.  Compare the actual and expected hashes, and print a
"hash-mismatch" status line when they differ.  When they match, print
not just "success" but also the nar hash and size.
* nix/libstore/build.cc (class SubstitutionGoal)[expectedHashStr]:
Remove.
(SubstitutionGoal::finished): Tokenize 'status'.  Parse it and handle
"success" and "hash-mismatch" accordingly.  Call 'hashPath' only when
the returned hash is not SHA256.
(SubstitutionGoal::handleChildOutput): Remove 'expectedHashStr'
handling.
* tests/substitute.scm ("substitute, invalid hash"): Rename to...
("substitute, invalid narinfo hash"): ... this.
("substitute, invalid hash"): New test.

											
										
										
											2020-12-13 21:46:03 +00:00
+								(test-equal "substitute, invalid hash"
 								  (string-append "hash-mismatch sha256 "
 								                 (bytevector->nix-base32-string (sha256 #vu8())) " "
 								                 (let-values (((port get-hash)
 								                               (open-hash-port (hash-algorithm sha256)))
 								                              ((content)
 								                               "Substitutable data."))
 								                   (write-file-tree "foo" port
 								                                    #:file-type+size
 								                                    (lambda _
 								                                      (values 'regular
 								                                              (string-length content)))
 								                                    #:file-port
 								                                    (lambda _
 								                                      (open-input-string content)))
 								                   (close-port port)
 								                   (bytevector->nix-base32-string (get-hash)))
 								                 "\n")
 								  ;; Arrange so the actual data hash does not match the 'NarHash' field in the
 								  ;; narinfo.
 								  (with-output-to-string
 								    (lambda ()
 								      (let ((narinfo (string-append "StorePath: " (%store-prefix)
 								                                    "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-wrong-hash
 								URL: example.nar
 								Compression: none
 								NarHash: sha256:" (bytevector->nix-base32-string (sha256 #vu8())) "
 								NarSize: 42
 								References:
 								Deriver: " (%store-prefix) "/foo.drv
 								System: mips64el-linux\n")))
 								        (with-narinfo (string-append narinfo "Signature: "
 								                                     (signature-field narinfo) "\n")
 								          (call-with-temporary-directory
 								           (lambda (directory)
 								             (with-input-from-string (string-append
 								                                      "substitute " (%store-prefix)
 								                                      "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-wrong-hash "
 								                                      directory "/wrong-hash\n")
 								               (lambda ()
 								                 (guix-substitute "--substitute"))))))))))
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								(test-quit "substitute, unauthorized key"
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								    "no valid substitute"
-												tests: Simplify 'substitute-binary' tests; reduce use of global variables.

* tests/substitute-binary.scm (signature-body): Change 'str' parameter
  to 'bv', and expect it to be a bytevector.
  (%signature-body, %wrong-signature, %acl): Remove.
  (signature): Rename to...
  (signature-field): ... this.  Add 'bv-or-str' parameter.  Change 'str'
  parameter to #:version.  Add #:public-key parameter.  Call
  'signature-body' directly.  Change domain part of the signature to
  'example.gnu.org'.
  ("not a number", "wrong version number", "valid
  narinfo-signature->canonical-sexp"): Use 'signature-field' instead of
  'signature' or %SIGNATURE.
  (test-error-condition): Add 'message-rx' parameter and honor it.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Adjust accordingly.
  (narinfo, %signed-narinfo): Remove.
  ("query narinfo with invalid hash"): Use '%narinfo' and
  'signature-field' instead of 'narinfo' and '%signature'.
  ("query narinfo signed with authorized key", "query narinfo signed
  with unauthorized key", "substitute, invalid hash", "substitute,
  unauthorized key"): Likewise.

											
										
										
											2014-03-30 19:35:22 +00:00
+								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field
 								                                %narinfo
 								                                #:public-key %wrong-public-key)
 								                               "\n")
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								    (with-input-from-string (string-append "substitute "
 								                                           (%store-prefix)
 								                                           "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
 								                                           " foo\n")
 								      (lambda ()
 								        (guix-substitute "--substitute")))))
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								(test-equal "substitute, authorized key"
-												tests: Check the mtime and permissions of substituted items.

* tests/store.scm ("substitute")
("substitute + build-things with output path")
("substitute + build-things with specific output"): Call 'canonical-file?'.
* tests/substitute.scm ("substitute, authorized key"): Check the mtime
and permissions of "substitute-retrieved".

											
										
										
											2020-12-14 13:33:06 +00:00
+								  '("Substitutable data." 1 #o444)
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field %narinfo))
 								    (dynamic-wind
 								      (const #t)
 								      (lambda ()
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								        (request-substitution (string-append (%store-prefix)
 								                                             "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                              "substitute-retrieved")
-												tests: Check the mtime and permissions of substituted items.

* tests/store.scm ("substitute")
("substitute + build-things with output path")
("substitute + build-things with specific output"): Call 'canonical-file?'.
* tests/substitute.scm ("substitute, authorized key"): Check the mtime
and permissions of "substitute-retrieved".

											
										
										
											2020-12-14 13:33:06 +00:00
+								        (list (call-with-input-file "substitute-retrieved" get-string-all)
 								              (stat:mtime (lstat "substitute-retrieved"))
 								              (stat:perms (lstat "substitute-retrieved"))))
-												tests: Add more signed substitute tests.

* tests/substitute-binary.scm (%narinfo): Set 'URL', 'Compression', and
  'NarHash'.
  (call-with-narinfo): Create 'example.out' and 'example.nar'.
  ("query narinfo without signature", "substitute, no signature",
  "substitute, authorized key"): New tests.

											
										
										
											2014-03-30 20:11:22 +00:00
+								      (lambda ()
 								        (false-if-exception (delete-file "substitute-retrieved"))))))
-												substitute: Test behavior with unroutable substitute server addresses.

* tests/substitute.scm (%unroutable-substitute-url): New variable.
("query narinfo signed with authorized key, unroutable URL first")
("substitute, authorized key, first substitute URL is unroutable"): New
tests.

											
										
										
											2022-09-21 15:21:48 +00:00
+								(test-equal "substitute, authorized key, first substitute URL is unroutable"
 								  '("Substitutable data." 1 #o444)
 								  (with-narinfo (string-append %narinfo "Signature: "
 								                               (signature-field %narinfo))
 								    (dynamic-wind
 								      (const #t)
 								      (lambda ()
 								        ;; Pick an unroutable URL as the first one.  This shouldn't be a
 								        ;; problem.
 								        (parameterize ((substitute-urls
 								                        (list %unroutable-substitute-url
 								                              (string-append "file://"
 								                                             %main-substitute-directory))))
 								          (request-substitution (string-append (%store-prefix)
 								                                               "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                "substitute-retrieved")
 								          (list (call-with-input-file "substitute-retrieved" get-string-all)
 								                (stat:mtime (lstat "substitute-retrieved"))
 								                (stat:perms (lstat "substitute-retrieved")))))
 								      (lambda ()
 								        (false-if-exception (delete-file "substitute-retrieved"))))))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								(test-equal "substitute, unauthorized narinfo comes first"
 								  "Substitutable data."
 								  (with-narinfo*
 								      (string-append %narinfo "Signature: "
 								                     (signature-field
 								                      %narinfo
 								                      #:public-key %wrong-public-key))
 								      %alternate-substitute-directory
 								    (with-narinfo* (string-append %narinfo "Signature: "
 								                                  (signature-field %narinfo))
 								        %main-substitute-directory
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          ;; Remove this file so that the substitute can only be retrieved
 								          ;; from %ALTERNATE-SUBSTITUTE-DIRECTORY.
 								          (delete-file (string-append %main-substitute-directory
 								                                      "/example.nar"))
 								          (parameterize ((substitute-urls
 								                          (map (cut string-append "file://" <>)
 								                               (list %alternate-substitute-directory
 								                                     %main-substitute-directory))))
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
 								(test-equal "substitute, unsigned narinfo comes first"
 								  "Substitutable data."
 								  (with-narinfo* %narinfo                         ;not signed!
 								      %alternate-substitute-directory
 								    (with-narinfo* (string-append %narinfo "Signature: "
 								                                  (signature-field %narinfo))
 								        %main-substitute-directory
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          ;; Remove this file so that the substitute can only be retrieved
 								          ;; from %ALTERNATE-SUBSTITUTE-DIRECTORY.
 								          (delete-file (string-append %main-substitute-directory
 								                                      "/example.nar"))
 								          (parameterize ((substitute-urls
 								                          (map (cut string-append "file://" <>)
 								                               (list %alternate-substitute-directory
 								                                     %main-substitute-directory))))
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
-												substitute: Retry downloading when a nar is unavailable.

Fixes <https://issues.guix.gnu.org/57978>
Reported by Attila Lendvai <attila@lendvai.name>.

Previously, if a narinfo was available but its corresponding nar was
missing (for instance because the narinfo was cached and the server
became unreachable in the meantime), 'guix substitute --substitute'
would try to download the nar from its preferred location and abort when
that fails.  This change forces one retry with each of the URLs.

* guix/scripts/substitute.scm (download-nar): Do not catch
'http-get-error?' exceptions.
(system-error?, network-error?, process-substitution/fallback): New
procedures.
(process-substitution): Call 'process-substitution/fallback' upon
'network-error?'.
* tests/substitute.scm ("substitute, first URL has narinfo but lacks nar, second URL unauthorized")
("substitute, first URL has narinfo but nar is 404, both URLs authorized")
("substitute, first URL has narinfo but nar is 404, one URL authorized")
("substitute, narinfo is available but nar is missing"): New tests.

											
										
										
											2022-09-23 06:00:13 +00:00
+								(test-equal "substitute, first URL has narinfo but lacks nar, second URL unauthorized"
 								  "Substitutable data."
 								  (with-narinfo*
 								      (string-append %narinfo "Signature: "
 								                     (signature-field
 								                      %narinfo
 								                      #:public-key %wrong-public-key))
 								      %alternate-substitute-directory
 								    (with-narinfo* (string-append %narinfo "Signature: "
 								                                  (signature-field %narinfo))
 								        %main-substitute-directory
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          ;; Remove this file so that the substitute can only be retrieved
 								          ;; from %ALTERNATE-SUBSTITUTE-DIRECTORY.
 								          (delete-file (string-append %main-substitute-directory
 								                                      "/example.nar"))
 								          (parameterize ((substitute-urls
 								                          (map (cut string-append "file://" <>)
 								                               (list %main-substitute-directory
 								                                     %alternate-substitute-directory))))
 								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
 								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
 								(test-equal "substitute, first URL has narinfo but nar is 404, both URLs authorized"
 								  "Substitutable data."
 								  (with-narinfo*
 								      (string-append %narinfo "Signature: "
 								                     (signature-field %narinfo))
 								      %main-substitute-directory
 								    (with-http-server `((200 ,(string-append %narinfo "Signature: "
 								                                             (signature-field %narinfo)))
 								                        (404 "Sorry, nar is missing!"))
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          (parameterize ((substitute-urls
 								                          (list (%local-url)
 								                                (string-append "file://"
 								                                               %main-substitute-directory))))
 								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
 								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
 								(test-equal "substitute, first URL has narinfo but nar is 404, one URL authorized"
 								  "Substitutable data."
 								  (with-narinfo*
 								      (string-append %narinfo "Signature: "
 								                     (signature-field
 								                      %narinfo
 								                      #:public-key %wrong-public-key))
 								      %main-substitute-directory
 								    (with-http-server `((200 ,(string-append %narinfo "Signature: "
 								                                             (signature-field
 								                                              %narinfo
 								                                              #:public-key %wrong-public-key)))
 								                        (404 "Sorry, nar is missing!"))
 								      (let ((url1 (%local-url)))
 								        (parameterize ((%http-server-port 0))
 								          (with-http-server `((200 ,(string-append %narinfo "Signature: "
 								                                                   (signature-field %narinfo)))
 								                              (404 "Sorry, nar is missing!"))
 								            (let ((url2 (%local-url)))
 								              (dynamic-wind
 								                (const #t)
 								                (lambda ()
 								                  (parameterize ((substitute-urls
 								                                  (list url1 url2
 								                                        (string-append "file://"
 								                                                       %main-substitute-directory))))
 								                    (request-substitution (string-append (%store-prefix)
 								                                                         "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                          "substitute-retrieved"))
 								                  (call-with-input-file "substitute-retrieved" get-string-all))
 								                (lambda ()
 								                  (false-if-exception (delete-file "substitute-retrieved")))))))))))
-												substitute: If a server's nar URL is 404, try the next one(s).

If a substitute server advertises in its narinfo, for example, both a
/zstd and a /lzip URL but the /zstd URL is unreachable, try the /lzip
URL.

Fixes <https://issues.guix.gnu.org/63634>.

* guix/narinfo.scm (narinfo-preferred-uris): New procedure.
(narinfo-best-uri): Rebase on top of it.
* guix/scripts/substitute.scm (download-nar)[try-fetch]: New procedure.
Use 'narinfo-preferred-uris' and 'try-fetch' to attempt all the URLs of
NARINFO.
* tests/substitute.scm (request-substitution): Remove 'parameterize'.
Delete DESTINATION.
("substitute, preferred nar URL is 404, other is 200"): New test.

											
										
										
											2023-05-22 15:19:39 +00:00
+								(test-equal "substitute, preferred nar URL is 404, other is 200"
 								  "Substitutable data."
 								  (with-narinfo* (string-append %narinfo "Signature: " (signature-field %narinfo))
 								      %main-substitute-directory
 								    (with-http-server `((200 ,(string-append %narinfo "Signature: "
 								                                             (signature-field %narinfo)
 								                                             "\n"
 								                                             "URL: example.nar.lz\n"
 								                                             "Compression: lzip\n"))
 								                        (404 "Sorry, nar.lz is missing!")
 								                        (200 ,(call-with-input-file
 								                                  (string-append %main-substitute-directory
 								                                                 "/example.nar")
 								                                get-bytevector-all)))
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          (parameterize ((substitute-urls (list (%local-url))))
 								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
 								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
-												substitute: Gracefully retry after failed partial downloads.

Fixes <https://issues.guix.gnu.org/63443>.
Reported by Attila Lendvai <attila@lendvai.name>.

* guix/scripts/substitute.scm (catch-system-error): New macro.
(download-nar): Add call to 'delete-file-recursively'.
* tests/substitute.scm ("substitute, previous partial download around"):
New test.

											
										
										
											2023-06-08 20:43:05 +00:00
+								(test-equal "substitute, previous partial download around"
 								  "Substitutable data."
 								  (with-narinfo* (string-append %narinfo "Signature: " (signature-field %narinfo))
 								      %main-substitute-directory
 								    (with-http-server `((200 ,(string-append %narinfo "Signature: "
 								                                             (signature-field %narinfo)))
 								                        (200 ,(call-with-input-file
 								                                  (string-append %main-substitute-directory
 								                                                 "/example.nar")
 								                                get-bytevector-all)))
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          (parameterize ((substitute-urls (list (%local-url))))
 								            (mkdir-p "substitute-retrieved/a/b/c/d") ;add stale data
 								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
 								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
-												substitute: Retry downloading when a nar is unavailable.

Fixes <https://issues.guix.gnu.org/57978>
Reported by Attila Lendvai <attila@lendvai.name>.

Previously, if a narinfo was available but its corresponding nar was
missing (for instance because the narinfo was cached and the server
became unreachable in the meantime), 'guix substitute --substitute'
would try to download the nar from its preferred location and abort when
that fails.  This change forces one retry with each of the URLs.

* guix/scripts/substitute.scm (download-nar): Do not catch
'http-get-error?' exceptions.
(system-error?, network-error?, process-substitution/fallback): New
procedures.
(process-substitution): Call 'process-substitution/fallback' upon
'network-error?'.
* tests/substitute.scm ("substitute, first URL has narinfo but lacks nar, second URL unauthorized")
("substitute, first URL has narinfo but nar is 404, both URLs authorized")
("substitute, first URL has narinfo but nar is 404, one URL authorized")
("substitute, narinfo is available but nar is missing"): New tests.

											
										
										
											2022-09-23 06:00:13 +00:00
+								(test-quit "substitute, narinfo is available but nar is missing"
 								    "failed to find alternative substitute"
 								  (with-narinfo*
 								      (string-append %narinfo "Signature: "
 								                     (signature-field
 								                      %narinfo
 								                      #:public-key %wrong-public-key))
 								      %main-substitute-directory
 								    (with-http-server `((200 ,(string-append %narinfo "Signature: "
 								                                             (signature-field %narinfo)))
 								                        (404 "Sorry, nar is missing!"))
 								      (parameterize ((substitute-urls
 								                      (list (%local-url)
 								                            (string-append "file://"
 								                                           %main-substitute-directory))))
 								        (delete-file (string-append %main-substitute-directory
 								                                    "/example.nar"))
 								        (request-substitution (string-append (%store-prefix)
 								                                             "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                              "substitute-retrieved")
 								        (not (file-exists? "substitute-retrieved"))))))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								(test-equal "substitute, first narinfo is unsigned and has wrong hash"
 								  "Substitutable data."
 								  (with-narinfo* (regexp-substitute #f
 								                                    (string-match "NarHash: [[:graph:]]+"
 								                                                  %narinfo)
 								                                    'pre
 								                                    "NarHash: sha256:"
 								                                    (bytevector->nix-base32-string
 								                                     (make-bytevector 32))
 								                                    'post)
 								      %alternate-substitute-directory
 								    (with-narinfo* (string-append %narinfo "Signature: "
 								                                  (signature-field %narinfo))
 								        %main-substitute-directory
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          ;; This time remove the file so that the substitute can only be
 								          ;; retrieved from %MAIN-SUBSTITUTE-DIRECTORY.
 								          (delete-file (string-append %alternate-substitute-directory
 								                                      "/example.nar"))
 								          (parameterize ((substitute-urls
 								                          (map (cut string-append "file://" <>)
 								                               (list %alternate-substitute-directory
 								                                     %main-substitute-directory))))
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
 								(test-equal "substitute, first narinfo is unsigned and has wrong refs"
 								  "Substitutable data."
 								  (with-narinfo* (regexp-substitute #f
 								                                    (string-match "References: ([^\n]+)\n"
 								                                                  %narinfo)
 								                                    'pre "References: " 1
 								                                    " wrong set of references\n"
 								                                    'post)
 								      %alternate-substitute-directory
 								    (with-narinfo* (string-append %narinfo "Signature: "
 								                                  (signature-field %narinfo))
 								        %main-substitute-directory
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          ;; This time remove the file so that the substitute can only be
 								          ;; retrieved from %MAIN-SUBSTITUTE-DIRECTORY.
 								          (delete-file (string-append %alternate-substitute-directory
 								                                      "/example.nar"))
 								          (parameterize ((substitute-urls
 								                          (map (cut string-append "file://" <>)
 								                               (list %alternate-substitute-directory
 								                                     %main-substitute-directory))))
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
 								(test-quit "substitute, two invalid narinfos"
 								    "no valid substitute"
 								  (with-narinfo* %narinfo                         ;not signed
 								      %alternate-substitute-directory
 								    (with-narinfo* (string-append %narinfo "Signature: " ;unauthorized
 								                                  (signature-field
 								                                   %narinfo
 								                                   #:public-key %wrong-public-key))
 								        %main-substitute-directory
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								      (with-input-from-string (string-append "substitute "
 								                                             (%store-prefix)
 								                                             "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"
 								                                             " substitute-retrieved\n")
 								        (lambda ()
 								          (guix-substitute "--substitute"))))))
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
-												substitute: Select the best compression methods.

When a server publishes several URLs with different compression methods,
'guix substitute' can now choose the best one among the compression
methods that it supports.

* guix/scripts/substitute.scm (<narinfo>)[uri]: Replace with...
[uris]: ... this.
[compression]: Replace with...
[compressions]: ... this.
[file-size]: Replace with...
[file-sizes]: ... this.
[file-hash]: Replace with...
[file-hashes]: ... this.
(narinfo-maker): Adjust accordingly.  Ensure 'file-sizes' and
'file-hashes' have the right length.
(assert-valid-signature, valid-narinfo?): Use the first element of
'narinfo-uris' in error messages.
(read-narinfo): Expect "URL", "Compression", "FileSize", and "FileHash"
to occur multiple times.
(display-narinfo-data): Call 'select-uri' to determine the file size.
(%compression-methods): New variable.
(supported-compression?, compresses-better?, select-uri): New
procedures.
(process-substitution): Call 'select-uri' to select the URI and
compression.
* guix/scripts/weather.scm (report-server-coverage): Account for all the
values returned by 'narinfo-file-sizes'.
* tests/substitute.scm ("substitute, narinfo with several URLs"): New
test.

											
										
										
											2019-05-31 14:26:08 +00:00
+								(test-equal "substitute, narinfo with several URLs"
 								  "Substitutable data."
 								  (let ((narinfo (string-append "StorePath: " (%store-prefix)
 								                                "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo
 								URL: example.nar.gz
 								Compression: gzip
 								URL: example.nar.lz
 								Compression: lzip
 								URL: example.nar
 								Compression: none
 								NarHash: sha256:" (bytevector->nix-base32-string
 								                   (sha256 (string->utf8 "Substitutable data."))) "
 								NarSize: 42
 								References: bar baz
 								Deriver: " (%store-prefix) "/foo.drv
 								System: mips64el-linux\n")))
 								    (with-narinfo (string-append narinfo "Signature: "
 								                                 (signature-field narinfo))
 								      (dynamic-wind
 								        (const #t)
 								        (lambda ()
 								          (define (compress input output compression)
 								            (call-with-output-file output
 								              (lambda (port)
 								                (call-with-compressed-output-port compression port
 								                  (lambda (port)
 								                    (call-with-input-file input
 								                      (lambda (input)
 								                        (dump-port input port))))))))
 								          (let ((nar (string-append %main-substitute-directory
 								                                    "/example.nar")))
 								            (compress nar (string-append nar ".gz") 'gzip)
-												Use "guile-zlib" and "guile-lzlib" instead of (guix config).

* Makefile.am (MODULES): Remove guix/zlib.scm and guix/lzlib.scm,
(SCM_TESTS): remove tests/zlib.scm, tests/lzlib.scm.
* build-aux/build-self.scm (make-config.scm): Remove unused %libz variable.
* configure.ac: Remove LIBZ and LIBLZ variables and check instead for
Guile-zlib and Guile-lzlib.
* doc/guix.texi ("Requirements"): Remove zlib requirement and add Guile-zlib
and Guile-lzlib instead.
* gnu/packages/package-management.scm (guix)[native-inputs]: Add "guile-zlib"
and "guile-lzlib",
[inputs]: remove "zlib" and "lzlib",
[propagated-inputs]: ditto,
[arguments]: add "guile-zlib" and "guile-lzlib" to Guile load path.
* guix/config.scm.in (%libz, %liblz): Remove them.
* guix/lzlib.scm: Remove it.
* guix/man-db.scm: Use (zlib) instead of (guix zlib).
* guix/profiles.scm (manual-database): Do not stub (guix config) in imported
modules list, instead add "guile-zlib" to the extension list.
* guix/scripts/publish.scm: Use (zlib) instead of (guix zlib) and (lzlib)
instead of (guix lzlib),
(string->compression-type, effective-compression): do not check for zlib and
lzlib availability.
* guix/scripts/substitute.scm (%compression-methods): Do not check for lzlib
availability.
* guix/self.scm (specification->package): Add "guile-zlib" and "guile-lzlib"
and remove "zlib" and "lzlib",
(compiled-guix): remove "zlib" and "lzlib" arguments and add guile-zlib and
guile-lzlib to the dependencies, also do not pass "zlib" and "lzlib" to
"make-config.scm" procedure,
(make-config.scm): remove "zlib" and "lzlib" arguments as well as %libz and
%liblz variables.
* guix/utils.scm (lzip-port): Use (lzlib) instead of (guix lzlib) and do not
check for lzlib availability.
* guix/zlib.scm: Remove it.
* m4/guix.m4 (GUIX_LIBZ_LIBDIR, GUIX_LIBLZ_FILE_NAME): Remove them.
* tests/lzlib.scm: Use (zlib) instead of (guix zlib) and (lzlib)
instead of (guix lzlib), and do not check for zlib and lzlib availability.
* tests/publish.scm: Ditto.
* tests/substitute.scm: Do not check for lzlib availability.
* tests/utils.scm: Ditto.
* tests/zlib.scm: Remove it.

											
										
										
											2020-07-27 14:36:39 +00:00
+								            (compress nar (string-append nar ".lz") 'lzip))
-												substitute: Select the best compression methods.

When a server publishes several URLs with different compression methods,
'guix substitute' can now choose the best one among the compression
methods that it supports.

* guix/scripts/substitute.scm (<narinfo>)[uri]: Replace with...
[uris]: ... this.
[compression]: Replace with...
[compressions]: ... this.
[file-size]: Replace with...
[file-sizes]: ... this.
[file-hash]: Replace with...
[file-hashes]: ... this.
(narinfo-maker): Adjust accordingly.  Ensure 'file-sizes' and
'file-hashes' have the right length.
(assert-valid-signature, valid-narinfo?): Use the first element of
'narinfo-uris' in error messages.
(read-narinfo): Expect "URL", "Compression", "FileSize", and "FileHash"
to occur multiple times.
(display-narinfo-data): Call 'select-uri' to determine the file size.
(%compression-methods): New variable.
(supported-compression?, compresses-better?, select-uri): New
procedures.
(process-substitution): Call 'select-uri' to select the URI and
compression.
* guix/scripts/weather.scm (report-server-coverage): Account for all the
values returned by 'narinfo-file-sizes'.
* tests/substitute.scm ("substitute, narinfo with several URLs"): New
test.

											
										
										
											2019-05-31 14:26:08 +00:00
 								          (parameterize ((substitute-urls
 								                          (list (string-append "file://"
 								                                               %main-substitute-directory))))
-												daemon: Run 'guix substitute --substitute' as an agent.

This avoids spawning one substitute process per substitution.

* nix/libstore/build.cc (class Worker)[substituter]: New field.
[outPipe, logPipe, pid]: Remove.
(class SubstitutionGoal)[expectedHashStr, status, substituter]: New fields.
(SubstitutionGoal::timedOut): Adjust to check 'substituter'.
(SubstitutionGoal::tryToRun): Remove references to 'outPipe' and
'logPipe'.  Run "guix substitute --substitute" as an 'Agent'.  Send the
request with 'writeLine'.
(SubstitutionGoal::finished): Likewise.
(SubstitutionGoal::handleChildOutput): Change to fill in
'expectedHashStr' and 'status'.
(SubstitutionGoal::handleEOF): Call 'wakeUp' unconditionally.
(SubstitutionGoal::~SubstitutionGoal): Adjust to check 'substituter'.
* guix/scripts/substitute.scm (process-substitution): Write "success\n"
to stdout upon success.
(%error-to-file-descriptor-4?): New variable.
(guix-substitute): Set 'current-error-port' to file descriptor 4
unless (%error-to-file-descriptor-4?) is false.
Remove "--substitute" arguments.  Loop reading line from stdin.
* tests/substitute.scm <top level>: Call '%error-to-file-descriptor-4?'.
(request-substitution): New procedure.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key")
("substitute, authorized key")
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, two invalid narinfos")
("substitute, narinfo with several URLs"): Adjust to new "guix
substitute --substitute" calling convention.

											
										
										
											2020-12-02 15:27:34 +00:00
+								            (request-substitution (string-append (%store-prefix)
 								                                                 "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo")
 								                                  "substitute-retrieved"))
-												substitute: Select the best compression methods.

When a server publishes several URLs with different compression methods,
'guix substitute' can now choose the best one among the compression
methods that it supports.

* guix/scripts/substitute.scm (<narinfo>)[uri]: Replace with...
[uris]: ... this.
[compression]: Replace with...
[compressions]: ... this.
[file-size]: Replace with...
[file-sizes]: ... this.
[file-hash]: Replace with...
[file-hashes]: ... this.
(narinfo-maker): Adjust accordingly.  Ensure 'file-sizes' and
'file-hashes' have the right length.
(assert-valid-signature, valid-narinfo?): Use the first element of
'narinfo-uris' in error messages.
(read-narinfo): Expect "URL", "Compression", "FileSize", and "FileHash"
to occur multiple times.
(display-narinfo-data): Call 'select-uri' to determine the file size.
(%compression-methods): New variable.
(supported-compression?, compresses-better?, select-uri): New
procedures.
(process-substitution): Call 'select-uri' to select the URI and
compression.
* guix/scripts/weather.scm (report-server-coverage): Account for all the
values returned by 'narinfo-file-sizes'.
* tests/substitute.scm ("substitute, narinfo with several URLs"): New
test.

											
										
										
											2019-05-31 14:26:08 +00:00
+								          (call-with-input-file "substitute-retrieved" get-string-all))
 								        (lambda ()
 								          (false-if-exception (delete-file "substitute-retrieved")))))))
-												Rename 'guix substitute-binary' to 'guix substitute'.

* guix/scripts/substitute-binary.scm: Rename to...
* guix/scripts/substitute.scm: ... this.  Adjust module name, entry
  point, comments, and help string accordingly.
* nix/scripts/substitute-binary.in: Rename to...
* nix/scripts/substitute.in: ... this.
* pre-inst-env.in (NIX_SUBSTITUTERS): Adjust accordingly.
* tests/substitute-binary.scm: Rename to...
* tests/substitute.scm: ... this.  Adjust references to (guix scripts
  substitute) accordingly.
* guix/ui.scm (show-guix-help)[internal?]: Change "substitute-binary" to
  "substitute".
* Makefile.am (MODULES, SCM_TESTS): Adjust to file renames.
* daemon.am (nodist_pkglibexec_SCRIPTS): Likewise.
* config-daemon.ac: Likewise.
* guix/tests.scm (call-with-derivation-narinfo): Adjust comments and
  docstring.

											
										
										
											2015-03-25 09:34:27 +00:00
+								(test-end "substitute")
-												substitute-binary: Support the Signature field of a narinfo file.

* guix/scripts/substitute-binary.scm (<narinfo>): Add the 'signature'
  and 'contents' fields.
  (narinfo-signature->canonical-sexp): New function.
  (narinfo-maker): Add the 'signature' argument and use it.
  (assert-valid-signature): New function.
  (read-narinfo): Support the Signature field.
  (write-narinfo): Use 'narinfo-contents'.
  (%allow-unauthenticated-substitutes?): New variable.
* guix/base64.scm, tests/base64.scm, tests/substitute-binary.scm: New files.
* Makefile.am (SCM_TESTS): Add tests/base64.scm and
  tests/substitute-binary.scm.
  (MODULES): Add guix/base64.scm.
* test-env.in: Set 'GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES'.

											
										
										
											2014-03-28 23:06:41 +00:00
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								;;; Local Variables:
 								;;; eval: (put 'with-narinfo 'scheme-indent-function 1)
-												substitute: Download from unauthorized sources that provide the right content.

This allows substitutes to be downloaded from unauthorized servers, as
long as they advertise the same hash and references as one of the
authorized servers.

* guix/scripts/substitute.scm (assert-valid-narinfo): Remove.
(valid-narinfo?): Add #:verbose?.  Handle each case of
'signature-case'.
(equivalent-narinfo?): New procedure.
(lookup-narinfos/diverse): Add 'authorized?' parameter and honor it.
[select-hit]: New procedure.
(lookup-narinfo): Add 'authorized?' parameter and pass it.
(process-query): Adjust callers accordingly.
(process-substitution): Remove call to 'assert-valid-narinfo'.  Check
whether 'lookup-narinfo' returns true and call 'leave' if not.
* tests/substitute.scm (%main-substitute-directory)
(%alternate-substitute-directory): New variables.
(call-with-narinfo): Make 'narinfo-directory' a parameter.  Call
'mkdir-p' to create it.  Change unwind handler to check whether
CACHE-DIRECTORY exists before deleting it.
(with-narinfo*): New macro.
("substitute, no signature")
("substitute, invalid hash")
("substitute, unauthorized key"): Change expected error message to "no
valid substitute".
("substitute, unauthorized narinfo comes first")
("substitute, unsigned narinfo comes first")
("substitute, first narinfo is unsigned and has wrong hash")
("substitute, first narinfo is unsigned and has wrong refs")
("substitute, unsigned narinfo comes first")
("substitute, two invalid narinfos"): New tests.
* doc/guix.texi (Substitutes): Explain the new behavior.

											
										
										
											2017-08-31 22:15:31 +00:00
+								;;; eval: (put 'with-narinfo* 'scheme-indent-function 2)
-												tests: Test the error output of 'substitute-binary'.

* tests/substitute-binary.scm (test-error*): Rename to...
  (test-quit): ... this.  Add 'error-rx' parameter and honor it.
  ("not a number", "wrong version number", "substitute, no signature",
  "substitute, invalid hash", "substitute, unauthorized key"): Adjust
  accordingly.

											
										
										
											2014-03-30 20:25:47 +00:00
+								;;; eval: (put 'test-quit 'scheme-indent-function 2)
-												substitute-binary: Defer narinfo authentication and authorization checks.

* guix/scripts/substitute-binary.scm (narinfo-signature->canonical-sexp):
  Catch 'gcry-error' around 'string->canonical-sexp' call, and re-raise
  as a SRFI-35 &message and &nar-signature-error.
  (narinfo-maker): Handle when SIGNATURE is #f or an invalid canonical
  sexp.
  (&nar-signature-error, &nar-invalid-hash-error): New variables.
  (assert-valid-signature): Use them.  Expect 'signature' to be a
  canonical sexp.
  (read-narinfo): Remove authentication and authorization checks.
  (%signature-line-rx): New variable.
  (assert-valid-narinfo, valid-narinfo?): New procedures.
  (guix-substitute-binary): Wrap body in 'with-error-handling'.
  [valid?]: New procedure.
  <--query>: Show only store items of narinfos that match
  'valid-narinfo?'.
  <--substitute>: Call 'assert-valid-narinfo'.
* tests/substitute-binary.scm (test-error*): Use 'test-equal'.
  (%keypair): Remove.
  (%public-key, %private-key): Load from signing-key.{pub,sec}.
  (signature-body): Add #:public-key parameter.
  (call-with-narinfo): New procedure.
  (with-narinfo): New macro.
  ("corrupt signature data", "unauthorized public key", "invalid
  signature"): Make the first argument to 'assert-valid-signature' a
  canonical sexp.
  ("invalid hash", "valid read-narinfo", "valid write-narinfo"):
  Remove.
  ("query narinfo with invalid hash", "query narinfo signed with
  authorized key", "query narinfo signed with unauthorized key",
  "substitute, invalid hash", "substitute, unauthorized key"): New
  tests.

											
										
										
											2014-03-30 20:29:35 +00:00
+								;;; End: