me
/
guix
Archived
1
0
Fork 0
Code

build: Makefile splits Scheme compilation in four steps. 

Fixes <https://bugs.gnu.org/48963>.
Reported by Julien Lepiller <julien@lepiller.eu>.

This reduces peak memory consumption to something less unreasonable.

* Makefile.am (make-go): Depend on 'make-*-go' targets; remove body.
(guile-compilation-rule): New function.
(MODULES_CORE, MODULES_PACKAGES, MODULES_SYSTEM, MODULES_CLI): New
variables.
<top level>: Call 'guile-compilation-rule' 4 times.
* build-aux/compile-all.scm <top level>: Expect "--total" and
"--processed".  Take them into account when displaying progress
reports.
master
Ludovic Courtès 2021-06-23 22:43:00 +02:00
parent a807d84921
commit ef82ba9dd9
No known key found for this signature in database
GPG Key ID: 090B11993D9AEBB5
2 changed files with 75 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
Makefile.am
View File

@ -663,15 +663,53 @@ CLEANFILES = \
# the whole thing. Likewise, set 'XDG_CACHE_HOME' to avoid loading possibly # the whole thing. Likewise, set 'XDG_CACHE_HOME' to avoid loading possibly
# stale files from ~/.cache/guile/ccache. # stale files from ~/.cache/guile/ccache.
%.go: make-go ; @: %.go: make-go ; @:
make-go: $(MODULES) guix/config.scm $(dist_noinst_DATA) make-go: make-core-go make-packages-go make-system-go make-cli-go
$(AM_V_at)echo "Compiling Scheme modules..." ; \
unset GUILE_LOAD_COMPILED_PATH ; \ # Define a rule to build a subset of the .go files.
XDG_CACHE_HOME=/nowhere \ define guile-compilation-rule
host=$(host) srcdir="$(top_srcdir)" \
$(top_builddir)/pre-inst-env \ $(1): $(2)
$(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)" \ $(AM_V_at)echo "Compiling Scheme modules..." ; \
--no-auto-compile \ unset GUILE_LOAD_COMPILED_PATH ; \
-s "$(top_srcdir)"/build-aux/compile-all.scm $^ XDG_CACHE_HOME=/nowhere \
host=$(host) srcdir="$(top_srcdir)" \
$(top_builddir)/pre-inst-env \
$(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)" \
--no-auto-compile \
-s "$(top_srcdir)"/build-aux/compile-all.scm \
--total $(words $(MODULES)) \
--completed $(3) \
$$(filter %.scm,$$^)
.PHONY: $(1)
endef
# Split compilation in several steps, each of which building a subset of
# $(MODULES). The main goal is to reduce peak memory consumption, as reported
# in <https://issues.guix.gnu.org/48963>. Each 'eval' call below creates a
# 'make-*-go' phony target that builds the corresponding subset.
MODULES_CORE = $(filter-out guix/scripts/%,$(filter guix/%,$(MODULES)))
MODULES_PACKAGES = $(filter gnu/packages/%,$(MODULES))
MODULES_SYSTEM = $(filter-out gnu/packages/%,$(filter gnu/%,$(MODULES)))
MODULES_CLI = $(filter guix/scripts/%,$(MODULES))
$(eval $(call guile-compilation-rule,make-core-go, \
$(MODULES_CORE) guix/config.scm $(dist_noinst_DATA), \
0))
$(eval $(call guile-compilation-rule,make-packages-go, \
$(MODULES_PACKAGES) make-core-go, \
$(words $(MODULES_CORE))))
$(eval $(call guile-compilation-rule,make-system-go, \
$(MODULES_SYSTEM) make-packages-go make-core-go, \
$(words $(MODULES_CORE) $(MODULES_PACKAGES))))
$(eval $(call guile-compilation-rule,make-cli-go, \
$(MODULES_CLI) make-system-go make-packages-go make-core-go, \
$(words $(MODULES_CORE) $(MODULES_PACKAGES) $(MODULES_SYSTEM))))
SUFFIXES = .go SUFFIXES = .go

46
build-aux/compile-all.scm
View File

@ -98,26 +98,36 @@ to 'make'."
(exit 1))) (exit 1)))
(match (command-line) (match (command-line)
((_ . files) ((_ "--total" (= string->number grand-total)
"--completed" (= string->number processed)
. files)
;; GRAND-TOTAL is the total number of .scm files in the project; PROCESSED
;; is the total number of .scm files already compiled in previous
;; invocations of this script.
(catch #t (catch #t
(lambda () (lambda ()
(compile-files srcdir (getcwd) (let* ((to-build (filter file-needs-compilation? files))
(filter file-needs-compilation? files) (processed (+ processed
#:workers (parallel-job-count*) (- (length files) (length to-build)))))
#:host host (compile-files srcdir (getcwd) to-build
#:report-load (lambda (file total completed) #:workers (parallel-job-count*)
(when file #:host host
(format #t "[~3d%] LOAD ~a~%" #:report-load (lambda (file total completed)
(% (+ 1 completed) (* 2 total)) (when file
file) (format #t "[~3d%] LOAD ~a~%"
(force-output))) (% (+ 1 completed
#:report-compilation (lambda (file total completed) (* 2 processed))
(when file (* 2 grand-total))
(format #t "[~3d%] GUILEC ~a~%" file)
(% (+ total completed 1) (force-output)))
(* 2 total)) #:report-compilation (lambda (file total completed)
(scm->go file)) (when file
(force-output))))) (format #t "[~3d%] GUILEC ~a~%"
(% (+ total completed 1
(* 2 processed))
(* 2 grand-total))
(scm->go file))
(force-output))))))
(lambda _ (lambda _
(primitive-exit 1)) (primitive-exit 1))
(lambda args (lambda args