me
/
guix
Archived
1
0
Fork 0

daemon: Use 'Agent' to spawn 'guix substitute --query'.

* nix/libstore/local-store.hh (RunningSubstituter): Remove.
(LocalStore)[runningSubstituter]: Change to unique_ptr<Agent>.
[setSubstituterEnv, didSetSubstituterEnv]: Remove.
[getLineFromSubstituter, getIntLineFromSubstituter]: Take an 'Agent'.
* nix/libstore/local-store.cc (LocalStore::~LocalStore): Remove
reference to 'runningSubstituter'.
(LocalStore::setSubstituterEnv, LocalStore::startSubstituter): Remove.
(LocalStore::getLineFromSubstituter): Adjust to 'run' being an 'Agent'.
(LocalStore::querySubstitutablePaths): Spawn substituter agent if
needed.  Adjust to 'Agent' interface.
(LocalStore::querySubstitutablePathInfos): Likewise.
* nix/libstore/build.cc (SubstitutionGoal::tryToRun): Remove call to
'setSubstituterEnv' and add 'setenv' call for "_NIX_OPTIONS" instead.
(SubstitutionGoal::finished): Remove 'readLine' call for 'dummy'.
* guix/scripts/substitute.scm (%allow-unauthenticated-substitutes?):
Remove second argument to 'make-parameter'.
(process-query): Call 'warn-about-missing-authentication'
when (%allow-unauthenticated-substitutes?) is #t.
(guix-substitute): Wrap body in 'parameterize'.  Set 'guix-warning-port'
too.  No longer exit when 'substitute-urls' returns the empty list.  No
longer print newline initially.
* tests/substitute.scm (test-quit): Parameterize 'current-error-port' to
account for the port changes in 'guix-substitute'.
master
Ludovic Courtès 2020-12-01 15:01:40 +01:00
parent 2e308238ad
commit 79c6614f58
No known key found for this signature in database
GPG Key ID: 090B11993D9AEBB5
5 changed files with 104 additions and 206 deletions

View File

@ -124,11 +124,7 @@ disabled!~%"))
;; purposes, and should be avoided otherwise.
(make-parameter
(and=> (getenv "GUIX_ALLOW_UNAUTHENTICATED_SUBSTITUTES")
(cut string-ci=? <> "yes"))
(lambda (value)
(when value
(warn-about-missing-authentication))
value)))
(cut string-ci=? <> "yes"))))
(define %narinfo-ttl
;; Number of seconds during which cached narinfo lookups are considered
@ -893,6 +889,9 @@ authorized substitutes."
(define (valid? obj)
(valid-narinfo? obj acl))
(when (%allow-unauthenticated-substitutes?)
(warn-about-missing-authentication))
(match (string-tokenize command)
(("have" paths ..1)
;; Return the subset of PATHS available in CACHE-URLS.
@ -1139,68 +1138,67 @@ default value."
((= string->number number) (> number 0))
(_ #f)))
(mkdir-p %narinfo-cache-directory)
(maybe-remove-expired-cache-entries %narinfo-cache-directory
cached-narinfo-files
#:entry-expiration
cached-narinfo-expiration-time
#:cleanup-period
%narinfo-expired-cache-entry-removal-delay)
(check-acl-initialized)
;; The daemon's agent code opens file descriptor 4 for us and this is where
;; stderr should go.
(parameterize ((current-error-port (match args
(("--query") (fdopen 4 "wl"))
(_ (current-error-port)))))
;; Redirect diagnostics to file descriptor 4 as well.
(guix-warning-port (current-error-port))
;; Starting from commit 22144afa in Nix, we are allowed to bail out directly
;; when we know we cannot substitute, but we must emit a newline on stdout
;; when everything is alright.
(when (null? (substitute-urls))
(exit 0))
(mkdir-p %narinfo-cache-directory)
(maybe-remove-expired-cache-entries %narinfo-cache-directory
cached-narinfo-files
#:entry-expiration
cached-narinfo-expiration-time
#:cleanup-period
%narinfo-expired-cache-entry-removal-delay)
(check-acl-initialized)
;; Say hello (see above.)
(newline)
(force-output (current-output-port))
;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error
;; message.
(for-each validate-uri (substitute-urls))
;; Sanity-check SUBSTITUTE-URLS so we can provide a meaningful error message.
(for-each validate-uri (substitute-urls))
;; Attempt to install the client's locale so that messages are suitably
;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default
;; so don't change it.
(match (or (find-daemon-option "untrusted-locale")
(find-daemon-option "locale"))
(#f #f)
(locale (false-if-exception (setlocale LC_MESSAGES locale))))
;; Attempt to install the client's locale so that messages are suitably
;; translated. LC_CTYPE must be a UTF-8 locale; it's the case by default so
;; don't change it.
(match (or (find-daemon-option "untrusted-locale")
(find-daemon-option "locale"))
(#f #f)
(locale (false-if-exception (setlocale LC_MESSAGES locale))))
(catch 'system-error
(lambda ()
(set-thread-name "guix substitute"))
(const #t)) ;GNU/Hurd lacks 'prctl'
(catch 'system-error
(lambda ()
(set-thread-name "guix substitute"))
(const #t)) ;GNU/Hurd lacks 'prctl'
(with-networking
(with-error-handling ; for signature errors
(match args
(("--query")
(let ((acl (current-acl)))
(let loop ((command (read-line)))
(or (eof-object? command)
(begin
(process-query command
#:cache-urls (substitute-urls)
#:acl acl)
(loop (read-line)))))))
(("--substitute" store-path destination)
;; Download STORE-PATH and add store it as a Nar in file DESTINATION.
;; Specify the number of columns of the terminal so the progress
;; report displays nicely.
(parameterize ((current-terminal-columns (client-terminal-columns)))
(process-substitution store-path destination
#:cache-urls (substitute-urls)
#:acl (current-acl)
#:print-build-trace? print-build-trace?)))
((or ("-V") ("--version"))
(show-version-and-exit "guix substitute"))
(("--help")
(show-help))
(opts
(leave (G_ "~a: unrecognized options~%") opts))))))
(with-networking
(with-error-handling ; for signature errors
(match args
(("--query")
(let ((acl (current-acl)))
(let loop ((command (read-line)))
(or (eof-object? command)
(begin
(process-query command
#:cache-urls (substitute-urls)
#:acl acl)
(loop (read-line)))))))
(("--substitute" store-path destination)
;; Download STORE-PATH and store it as a Nar in file DESTINATION.
;; Specify the number of columns of the terminal so the progress
;; report displays nicely.
(parameterize ((current-terminal-columns (client-terminal-columns)))
(process-substitution store-path destination
#:cache-urls (substitute-urls)
#:acl (current-acl)
#:print-build-trace? print-build-trace?)))
((or ("-V") ("--version"))
(show-version-and-exit "guix substitute"))
(("--help")
(show-help))
(opts
(leave (G_ "~a: unrecognized options~%") opts)))))))
;;; Local Variables:
;;; eval: (put 'with-timeout 'scheme-indent-function 1)

View File

@ -2986,8 +2986,6 @@ void SubstitutionGoal::tryToRun()
if (pathExists(destPath))
deletePath(destPath);
worker.store.setSubstituterEnv();
/* Fill in the arguments. */
Strings args;
args.push_back("guix");
@ -2999,6 +2997,9 @@ void SubstitutionGoal::tryToRun()
/* Fork the substitute program. */
pid = startProcess([&]() {
/* Communicate substitute-urls & co. to 'guix substitute'. */
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
commonChildInit(logPipe);
if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
@ -3041,7 +3042,6 @@ void SubstitutionGoal::finished()
logPipe.readSide.close();
/* Get the hash info from stdout. */
string dummy = readLine(outPipe.readSide);
string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
outPipe.readSide.close();

View File

@ -57,7 +57,6 @@ void checkStoreNotSymlink()
LocalStore::LocalStore(bool reserveSpace)
: didSetSubstituterEnv(false)
{
schemaPath = settings.nixDBPath + "/schema";
@ -182,21 +181,6 @@ LocalStore::LocalStore(bool reserveSpace)
LocalStore::~LocalStore()
{
try {
if (runningSubstituter) {
RunningSubstituter &i = *runningSubstituter;
if (!i.disabled) {
i.to.close();
i.from.close();
i.error.close();
if (i.pid != -1)
i.pid.wait(true);
}
}
} catch (...) {
ignoreException();
}
try {
if (fdTempRoots != -1) {
fdTempRoots.close();
@ -796,96 +780,31 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
});
}
void LocalStore::setSubstituterEnv()
{
if (didSetSubstituterEnv) return;
/* Pass configuration options (including those overridden with
--option) to substituters. */
setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
didSetSubstituterEnv = true;
}
void LocalStore::startSubstituter(RunningSubstituter & run)
{
if (run.disabled || run.pid != -1) return;
debug(format("starting substituter program `%1% substitute'")
% settings.guixProgram);
Pipe toPipe, fromPipe, errorPipe;
toPipe.create();
fromPipe.create();
errorPipe.create();
setSubstituterEnv();
run.pid = startProcess([&]() {
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
throw SysError("dupping stdin");
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
throw SysError("dupping stdout");
if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
throw SysError("dupping stderr");
execl(settings.guixProgram.c_str(), "guix", "substitute", "--query", NULL);
throw SysError(format("executing `%1%'") % settings.guixProgram);
});
run.to = toPipe.writeSide.borrow();
run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
run.error = errorPipe.readSide.borrow();
toPipe.readSide.close();
fromPipe.writeSide.close();
errorPipe.writeSide.close();
/* The substituter may exit right away if it's disabled in any way
(e.g. copy-from-other-stores.pl will exit if no other stores
are configured). */
try {
getLineFromSubstituter(run);
} catch (EndOfFile & e) {
run.to.close();
run.from.close();
run.error.close();
run.disabled = true;
if (run.pid.wait(true) != 0) throw;
}
}
/* Read a line from the substituter's stdout, while also processing
its stderr. */
string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
string LocalStore::getLineFromSubstituter(Agent & run)
{
string res, err;
/* We might have stdout data left over from the last time. */
if (run.fromBuf.hasData()) goto haveData;
while (1) {
checkInterrupt();
fd_set fds;
FD_ZERO(&fds);
FD_SET(run.from, &fds);
FD_SET(run.error, &fds);
FD_SET(run.fromAgent.readSide, &fds);
FD_SET(run.builderOut.readSide, &fds);
/* Wait for data to appear on the substituter's stdout or
stderr. */
if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) {
if (select(std::max(run.fromAgent.readSide, run.builderOut.readSide) + 1, &fds, 0, 0, 0) == -1) {
if (errno == EINTR) continue;
throw SysError("waiting for input from the substituter");
}
/* Completely drain stderr before dealing with stdout. */
if (FD_ISSET(run.error, &fds)) {
if (FD_ISSET(run.builderOut.readSide, &fds)) {
char buf[4096];
ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf));
ssize_t n = read(run.builderOut.readSide, (unsigned char *) buf, sizeof(buf));
if (n == -1) {
if (errno == EINTR) continue;
throw SysError("reading from substituter's stderr");
@ -903,23 +822,20 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
}
/* Read from stdout until we get a newline or the buffer is empty. */
else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) {
haveData:
do {
unsigned char c;
run.fromBuf(&c, 1);
if (c == '\n') {
if (!err.empty()) printMsg(lvlError, "substitute: " + err);
return res;
}
res += c;
} while (run.fromBuf.hasData());
else if (FD_ISSET(run.fromAgent.readSide, &fds)) {
unsigned char c;
readFull(run.fromAgent.readSide, (unsigned char *) &c, 1);
if (c == '\n') {
if (!err.empty()) printMsg(lvlError, "substitute: " + err);
return res;
}
res += c;
}
}
}
template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run)
template<class T> T LocalStore::getIntLineFromSubstituter(Agent & run)
{
string s = getLineFromSubstituter(run);
T res;
@ -935,27 +851,26 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
if (!settings.useSubstitutes || paths.empty()) return res;
if (!runningSubstituter) {
std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
const Strings args = { "substitute", "--query" };
const std::map<string, string> env = { { "_NIX_OPTIONS", settings.pack() } };
std::unique_ptr<Agent> fresh(new Agent(settings.guixProgram, args, env));
runningSubstituter.swap(fresh);
}
RunningSubstituter & run = *runningSubstituter;
startSubstituter(run);
Agent & run = *runningSubstituter;
if (!run.disabled) {
string s = "have ";
foreach (PathSet::const_iterator, j, paths)
if (res.find(*j) == res.end()) { s += *j; s += " "; }
writeLine(run.to, s);
while (true) {
/* FIXME: we only read stderr when an error occurs, so
substituters should only write (short) messages to
stderr when they fail. I.e. they shouldn't write debug
output. */
Path path = getLineFromSubstituter(run);
if (path == "") break;
res.insert(path);
}
string s = "have ";
foreach (PathSet::const_iterator, j, paths)
if (res.find(*j) == res.end()) { s += *j; s += " "; }
writeLine(run.toAgent.writeSide, s);
while (true) {
/* FIXME: we only read stderr when an error occurs, so
substituters should only write (short) messages to
stderr when they fail. I.e. they shouldn't write debug
output. */
Path path = getLineFromSubstituter(run);
if (path == "") break;
res.insert(path);
}
return res;
@ -967,18 +882,18 @@ void LocalStore::querySubstitutablePathInfos(PathSet & paths, SubstitutablePathI
if (!settings.useSubstitutes) return;
if (!runningSubstituter) {
std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
const Strings args = { "substitute", "--query" };
const std::map<string, string> env = { { "_NIX_OPTIONS", settings.pack() } };
std::unique_ptr<Agent> fresh(new Agent(settings.guixProgram, args, env));
runningSubstituter.swap(fresh);
}
RunningSubstituter & run = *runningSubstituter;
startSubstituter(run);
if (run.disabled) return;
Agent & run = *runningSubstituter;
string s = "info ";
foreach (PathSet::const_iterator, i, paths)
if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
writeLine(run.to, s);
writeLine(run.toAgent.writeSide, s);
while (true) {
Path path = getLineFromSubstituter(run);

View File

@ -38,21 +38,11 @@ struct OptimiseStats
};
struct RunningSubstituter
{
Pid pid;
AutoCloseFD to, from, error;
FdSource fromBuf;
bool disabled;
RunningSubstituter() : disabled(false) { };
};
class LocalStore : public StoreAPI
{
private:
/* The currently running substituter or empty. */
std::unique_ptr<RunningSubstituter> runningSubstituter;
std::unique_ptr<Agent> runningSubstituter;
Path linksDir;
@ -178,8 +168,6 @@ public:
void markContentsGood(const Path & path);
void setSubstituterEnv();
void createUser(const std::string & userName, uid_t userId);
private:
@ -213,8 +201,6 @@ private:
/* Cache for pathContentsGood(). */
std::map<Path, bool> pathContentsGoodCache;
bool didSetSubstituterEnv;
/* The file to which we write our temporary roots. */
Path fnTempRoots;
AutoCloseFD fdTempRoots;
@ -262,11 +248,9 @@ private:
void removeUnusedLinks(const GCState & state);
void startSubstituter(RunningSubstituter & runningSubstituter);
string getLineFromSubstituter(Agent & run);
string getLineFromSubstituter(RunningSubstituter & run);
template<class T> T getIntLineFromSubstituter(RunningSubstituter & run);
template<class T> T getIntLineFromSubstituter(Agent & run);
Path createTempDirInStore();

View File

@ -47,7 +47,8 @@ it writes to GUIX-WARNING-PORT a messages that matches ERROR-RX."
(test-equal name
'(1 #t)
(let ((error-output (open-output-string)))
(parameterize ((guix-warning-port error-output))
(parameterize ((current-error-port error-output)
(guix-warning-port error-output))
(catch 'quit
(lambda ()
exp