me
/
guix
Archived
1
0
Fork 0
Code

transformations: Add '--tune'. 

* guix/transformations.scm (tuning-compiler)
(tuned-package, tunable-package?, package-tuning)
(transform-package-tuning)
(build-system-with-tuning-compiler): New procedures.
(%transformations): Add 'tune'.
(%transformation-options): Add "--tune".
* tests/transformations.scm ("options->transformation, tune")
("options->transformations, tune, wrong micro-architecture"): New
tests.
* doc/guix.texi (Package Transformation Options): Document '--tune'.
master
Ludovic Courtès 2021-12-03 23:06:49 +01:00 committed by Ludovic Courtès
parent 0a767f02d4
commit d090e9c37d
No known key found for this signature in database
GPG Key ID: 090B11993D9AEBB5
3 changed files with 300 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
doc/guix.texi
View File

@ -11015,6 +11015,67 @@ available options and a synopsis (these options are not shown in the
@table @code
@cindex performance, tuning code
@cindex optimization, of package code
@cindex tuning, of package code
@cindex SIMD support
@cindex tunable packages
@cindex package multi-versioning
@item --tune[=@var{cpu}]
Use versions of the packages marked as ``tunable'' optimized for
@var{cpu}. When @var{cpu} is @code{native}, or when it is omitted, tune
for the CPU on which the @command{guix} command is running.
Valid @var{cpu} names are those recognized by the underlying compiler,
by default the GNU Compiler Collection. On x86_64 processors, this
includes CPU names such as @code{nehalem}, @code{haswell}, and
@code{skylake} (@pxref{x86 Options, @code{-march},, gcc, Using the GNU
Compiler Collection (GCC)}).
As new generations of CPUs come out, they augment the standard
instruction set architecture (ISA) with additional instructions, in
particular instructions for single-instruction/multiple-data (SIMD)
parallel processing. For example, while Core2 and Skylake CPUs both
implement the x86_64 ISA, only the latter supports AVX2 SIMD
instructions.
The primary gain one can expect from @option{--tune} is for programs
that can make use of those SIMD capabilities @emph{and} that do not
already have a mechanism to select the right optimized code at run time.
Packages that have the @code{tunable?} property set are considered
@dfn{tunable packages} by the @option{--tune} option; a package
definition with the property set looks like this:
@lisp
(package
(name "hello-simd")
;; ...
;; This package may benefit from SIMD extensions so
;; mark it as "tunable".
(properties '((tunable? . #t))))
@end lisp
Other packages are not considered tunable. This allows Guix to use
generic binaries in the cases where tuning for a specific CPU is
unlikely to provide any gain.
Tuned packages are built with @code{-march=@var{CPU}}; under the hood,
the @option{-march} option is passed to the actual wrapper by a compiler
wrapper. Since the build machine may not be able to run code for the
target CPU micro-architecture, the test suite is not run when building a
tuned package.
To reduce rebuilds to the minimum, tuned packages are @emph{grafted}
onto packages that depend on them (@pxref{Security Updates, grafts}).
Thus, using @option{--no-grafts} cancels the effect of @option{--tune}.
We call this technique @dfn{package multi-versioning}: several variants
of tunable packages may be built, one for each CPU variant. It is the
coarse-grain counterpart of @dfn{function multi-versioning} as
implemented by the GNU tool chain (@pxref{Function Multiversioning,,,
gcc, Using the GNU Compiler Collection (GCC)}).
@item --with-source=@var{source}
@itemx --with-source=@var{package}=@var{source}
@itemx --with-source=@var{package}@@@var{version}=@var{source}

204
guix/transformations.scm
View File

@ -18,9 +18,11 @@
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
(define-module (guix transformations)
#:use-module ((guix config) #:select (%system))
#:use-module (guix i18n)
#:use-module (guix store)
#:use-module (guix packages)
#:use-module (guix build-system)
#:use-module (guix profiles)
#:use-module (guix diagnostics)
#:autoload (guix download) (download-to-store)
@ -29,6 +31,7 @@
#:autoload (guix upstream) (package-latest-release
upstream-source-version
upstream-source-signature-urls)
#:autoload (guix cpu) (current-cpu cpu->gcc-architecture)
#:use-module (guix utils)
#:use-module (guix memoization)
#:use-module (guix gexp)
@ -49,6 +52,9 @@
#:export (options->transformation
manifest-entry-with-transformations
tunable-package?
tuned-package
show-transformation-options-help
%transformation-options))
@ -419,6 +425,181 @@ the equal sign."
obj)
obj)))
(define tuning-compiler
(mlambda (micro-architecture)
"Return a compiler wrapper that passes '-march=MICRO-ARCHITECTURE' to the
actual compiler."
(define wrapper
#~(begin
(use-modules (ice-9 match))
(define* (search-next command
#:optional
(path (string-split (getenv "PATH")
#\:)))
;; Search the next COMMAND on PATH, a list of
;; directories representing the executable search path.
(define this
(stat (car (command-line))))
(let loop ((path path))
(match path
(()
(match command
("cc" (search-next "gcc"))
(_ #f)))
((directory rest ...)
(let* ((file (string-append
directory "/" command))
(st (stat file #f)))
(if (and st (not (equal? this st)))
file
(loop rest)))))))
(match (command-line)
((command arguments ...)
(match (search-next (basename command))
(#f (exit 127))
(next
(apply execl next
(append (cons next arguments)
(list (string-append "-march="
#$micro-architecture))))))))))
(define program
(program-file (string-append "tuning-compiler-wrapper-" micro-architecture)
wrapper))
(computed-file (string-append "tuning-compiler-" micro-architecture)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
(define bin (string-append #$output "/bin"))
(mkdir-p bin)
(for-each (lambda (program)
(symlink #$program
(string-append bin "/" program)))
'("cc" "gcc" "clang" "g++" "c++" "clang++")))))))
(define (build-system-with-tuning-compiler bs micro-architecture)
"Return a variant of BS, a build system, that ensures that the compiler that
BS uses (usually an implicit input) can generate code for MICRO-ARCHITECTURE,
which names a specific CPU of the target architecture--e.g., when targeting
86_64 MICRO-ARCHITECTURE might be \"skylake\". If it does, return a build
system that builds code for MICRO-ARCHITECTURE; otherwise raise an error."
(define %not-hyphen
(char-set-complement (char-set #\-)))
(define lower
(build-system-lower bs))
(define (lower* . args)
;; The list of CPU names supported by the '-march' option of C/C++
;; compilers is specific to each compiler and version thereof. Rather
;; than pass '-march=MICRO-ARCHITECTURE' as is to the compiler, possibly
;; leading to an obscure build error, check whether the compiler is known
;; to support MICRO-ARCHITECTURE. If not, bail out.
(let* ((lowered (apply lower args))
(architecture (match (string-tokenize (bag-system lowered)
%not-hyphen)
((arch _ ...) arch)))
(compiler (any (match-lambda
((label (? package? p) . _)
(and (assoc-ref (package-properties p)
'compiler-cpu-architectures)
p))
(_ #f))
(bag-build-inputs lowered))))
(unless compiler
(raise (formatted-message
(G_ "failed to determine which compiler is used"))))
(let ((lst (assoc-ref (package-properties compiler)
'compiler-cpu-architectures)))
(unless lst
(raise (formatted-message
(G_ "failed to determine whether ~a supports ~a")
(package-full-name compiler)
micro-architecture)))
(unless (member micro-architecture
(or (assoc-ref lst architecture) '()))
(raise (formatted-message
(G_ "compiler ~a does not support micro-architecture ~a")
(package-full-name compiler)
micro-architecture))))
(bag
(inherit lowered)
(build-inputs
;; Arrange so that the compiler wrapper comes first in $PATH.
`(("tuning-compiler" ,(tuning-compiler micro-architecture))
,@(bag-build-inputs lowered))))))
(build-system
(inherit bs)
(lower lower*)))
(define (tuned-package p micro-architecture)
"Return package P tuned for MICRO-ARCHITECTURE."
(package
(inherit p)
(build-system
(build-system-with-tuning-compiler (package-build-system p)
micro-architecture))
(arguments
;; The machine building this package may or may not be able to run code
;; for MICRO-ARCHITECTURE. Because of that, skip tests; they are run for
;; the "baseline" variant anyway.
(substitute-keyword-arguments (package-arguments p)
((#:tests? _ #f) #f)))
(properties
`((cpu-tuning . ,micro-architecture)
;; Remove the 'tunable?' property so that 'package-tuning' does not
;; call 'tuned-package' again on this one.
,@(alist-delete 'tunable? (package-properties p))))))
(define (tunable-package? package)
"Return true if package PACKAGE is \"tunable\"--i.e., if tuning it for the
host CPU is worthwhile."
(assq 'tunable? (package-properties package)))
(define package-tuning
(mlambda (micro-architecture)
"Return a procedure that maps the given package to its counterpart tuned
for MICRO-ARCHITECTURE, a string suitable for GCC's '-march'."
(define rewriting-property
(gensym " package-tuning"))
(package-mapping (lambda (p)
(cond ((assq rewriting-property (package-properties p))
p)
((assq 'tunable? (package-properties p))
(info (G_ "tuning ~a for CPU ~a~%")
(package-full-name p) micro-architecture)
(package/inherit p
(replacement (tuned-package p micro-architecture))
(properties `((,rewriting-property . #t)
,@(package-properties p)))))
(else
p)))
(lambda (p)
(assq rewriting-property (package-properties p)))
#:deep? #t)))
(define (transform-package-tuning micro-architectures)
"Return a procedure that, when "
(match micro-architectures
((micro-architecture _ ...)
(let ((rewrite (package-tuning micro-architecture)))
(lambda (obj)
(if (package? obj)
(rewrite obj)
obj))))))
(define (transform-package-with-debug-info specs)
"Return a procedure that, when passed a package, set its 'replacement' field
to the same package but with #:strip-binaries? #f in its 'arguments' field."
@ -601,6 +782,7 @@ are replaced by their latest upstream version."
(with-commit . ,transform-package-source-commit)
(with-git-url . ,transform-package-source-git-url)
(with-c-toolchain . ,transform-package-toolchain)
(tune . ,transform-package-tuning)
(with-debug-info . ,transform-package-with-debug-info)
(without-tests . ,transform-package-tests)
(with-patch . ,transform-package-patches)
@ -640,6 +822,28 @@ are replaced by their latest upstream version."
(parser 'with-git-url))
(option '("with-c-toolchain") #t #f
(parser 'with-c-toolchain))
(option '("tune") #f #t
(lambda (opt name arg result . rest)
(define micro-architecture
(match arg
((or #f "native")
(unless (string=? (or (assoc-ref result 'system)
(%current-system))
%system)
(leave (G_ "\
building for ~a instead of ~a, so tuning cannot be guessed~%")
(assoc-ref result 'system) %system))
(cpu->gcc-architecture (current-cpu)))
("generic" #f)
(_ arg)))
(apply values
(if micro-architecture
(alist-cons 'tune micro-architecture
result)
(alist-delete 'tune result))
rest)))
(option '("with-debug-info") #t #f
(parser 'with-debug-info))
(option '("without-tests") #t #f

35
tests/transformations.scm
View File

@ -38,12 +38,14 @@
#:use-module (guix utils)
#:use-module (guix git)
#:use-module (guix upstream)
#:use-module (guix diagnostics)
#:use-module (gnu packages)
#:use-module (gnu packages base)
#:use-module (gnu packages busybox)
#:use-module (ice-9 match)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-26)
#:use-module (srfi srfi-34)
#:use-module (srfi srfi-64))
@ -465,6 +467,39 @@
`((with-latest . "foo")))))
(package-version (t p)))))
(test-equal "options->transformation, tune"
'(cpu-tuning . "superfast")
(let* ((p0 (dummy-package "p0"))
(p1 (dummy-package "p1"
(inputs `(("p0" ,p0)))
(properties '((tunable? . #t)))))
(p2 (dummy-package "p2"
(inputs `(("p1" ,p1)))))
(t (options->transformation '((tune . "superfast"))))
(p3 (t p2)))
(and (not (package-replacement p3))
(match (package-inputs p3)
((("p1" tuned))
(match (package-inputs tuned)
((("p0" p0))
(and (not (package-replacement p0))
(assq 'cpu-tuning
(package-properties
(package-replacement tuned)))))))))))
(test-assert "options->transformations, tune, wrong micro-architecture"
(let ((p (dummy-package "tunable"
(properties '((tunable? . #t)))))
(t (options->transformation '((tune . "nonexistent-superfast")))))
;; Because GCC used by P's build system does not support
;; '-march=nonexistent-superfast', we should see an error when lowering
;; the tuned package.
(guard (c ((formatted-message? c)
(member "nonexistent-superfast"
(formatted-message-arguments c))))
(package->bag (t p))
#f)))
(test-equal "options->transformation + package->manifest-entry"
'((transformations . ((without-tests . "foo"))))
(let* ((p (dummy-package "foo"))