From 38f9de1a3d23e42345dea1801b05c96e63fdd656 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Fri, 5 Jan 2024 08:04:02 -0500 Subject: [PATCH 01/52] rename timer_end to timer_read --- include/platform/timer.h | 2 +- src/cmd_test.c | 4 ++-- src/platform/posix/timer.c | 2 +- src/platform/windows/timer.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/platform/timer.h b/include/platform/timer.h index d2bff8286..fb40ac031 100644 --- a/include/platform/timer.h +++ b/include/platform/timer.h @@ -28,7 +28,7 @@ struct timer { }; void timer_start(struct timer *t); -float timer_end(struct timer *t); +float timer_read(struct timer *t); void timer_sleep(uint64_t nanoseconds); #endif diff --git a/src/cmd_test.c b/src/cmd_test.c index 46573dbf1..42ce1133f 100644 --- a/src/cmd_test.c +++ b/src/cmd_test.c @@ -531,7 +531,7 @@ collect_tests(struct workspace *wk, struct run_test_ctx *ctx) } struct test_result *res = &ctx->jobs[i]; - res->dur = timer_end(&res->t); + res->dur = timer_read(&res->t); enum run_cmd_state state = run_cmd_collect(&res->cmd_ctx); @@ -671,7 +671,7 @@ push_test(struct workspace *wk, struct run_test_ctx *ctx, struct obj_test *test, res->busy = false; --ctx->busy_jobs; - res->dur = timer_end(&res->t); + res->dur = timer_read(&res->t); res->status = test_result_status_failed; print_test_progress(wk, ctx, res, true); arr_push(&ctx->test_results, res); diff --git a/src/platform/posix/timer.c b/src/platform/posix/timer.c index f2d9398de..4ae820d95 100644 --- a/src/platform/posix/timer.c +++ b/src/platform/posix/timer.c @@ -23,7 +23,7 @@ timer_start(struct timer *t) } float -timer_end(struct timer *t) +timer_read(struct timer *t) { struct timespec end; diff --git a/src/platform/windows/timer.c b/src/platform/windows/timer.c index db4233d01..81985bd84 100644 --- a/src/platform/windows/timer.c +++ b/src/platform/windows/timer.c @@ -16,7 +16,7 @@ timer_start(struct timer *t) } float -timer_end(struct timer *t) +timer_read(struct timer *t) { LARGE_INTEGER end; From eed03460ba210850f254fe9f0aedd72966d8d00c Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Fri, 5 Jan 2024 08:11:45 -0500 Subject: [PATCH 02/52] integrate samurai Copy samurai's sources into muon's tree and refactor slightly To prevent name collisions during bootstrap: - prefix all identifiers with samu_ or SAMU_ To make reentrant: - move all global state into a struct that gets passed around (samu_ctx) - convert all memory allocations to use an arena --- include/external/samurai/arg.h | 30 ++ include/external/samurai/build.h | 19 + include/external/samurai/ctx.h | 221 +++++++++++ include/external/samurai/deps.h | 17 + include/external/samurai/env.h | 42 +++ include/external/samurai/graph.h | 42 +++ include/external/samurai/htab.h | 21 ++ include/external/samurai/log.h | 16 + include/external/samurai/parse.h | 25 ++ include/external/samurai/samu.h | 9 + include/external/samurai/scan.h | 23 ++ include/external/samurai/tool.h | 10 + include/external/samurai/tree.h | 21 ++ include/external/samurai/util.h | 39 ++ src/amalgam.c | 16 +- src/backend/ninja.c | 15 +- src/external/meson.build | 23 +- src/external/samurai.c | 3 +- src/external/samurai/build.c | 624 +++++++++++++++++++++++++++++++ src/external/samurai/deps.c | 500 +++++++++++++++++++++++++ src/external/samurai/env.c | 261 +++++++++++++ src/external/samurai/graph.c | 233 ++++++++++++ src/external/samurai/htab.c | 169 +++++++++ src/external/samurai/log.c | 171 +++++++++ src/external/samurai/parse.c | 282 ++++++++++++++ src/external/samurai/samu.c | 300 +++++++++++++++ src/external/samurai/scan.c | 362 ++++++++++++++++++ src/external/samurai/tool.c | 475 +++++++++++++++++++++++ src/external/samurai/tree.c | 130 +++++++ src/external/samurai/util.c | 314 ++++++++++++++++ 30 files changed, 4395 insertions(+), 18 deletions(-) create mode 100644 include/external/samurai/arg.h create mode 100644 include/external/samurai/build.h create mode 100644 include/external/samurai/ctx.h create mode 100644 include/external/samurai/deps.h create mode 100644 include/external/samurai/env.h create mode 100644 include/external/samurai/graph.h create mode 100644 include/external/samurai/htab.h create mode 100644 include/external/samurai/log.h create mode 100644 include/external/samurai/parse.h create mode 100644 include/external/samurai/samu.h create mode 100644 include/external/samurai/scan.h create mode 100644 include/external/samurai/tool.h create mode 100644 include/external/samurai/tree.h create mode 100644 include/external/samurai/util.h create mode 100644 src/external/samurai/build.c create mode 100644 src/external/samurai/deps.c create mode 100644 src/external/samurai/env.c create mode 100644 src/external/samurai/graph.c create mode 100644 src/external/samurai/htab.c create mode 100644 src/external/samurai/log.c create mode 100644 src/external/samurai/parse.c create mode 100644 src/external/samurai/samu.c create mode 100644 src/external/samurai/scan.c create mode 100644 src/external/samurai/tool.c create mode 100644 src/external/samurai/tree.c create mode 100644 src/external/samurai/util.c diff --git a/include/external/samurai/arg.h b/include/external/samurai/arg.h new file mode 100644 index 000000000..973946f62 --- /dev/null +++ b/include/external/samurai/arg.h @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_ARG_H +#define MUON_EXTERNAL_SAMU_ARG_H + +#define SAMU_ARGBEGIN \ + for (;;) { \ + if (argc > 0) \ + ++argv, --argc; \ + if (argc == 0 || (*argv)[0] != '-') \ + break; \ + if ((*argv)[1] == '-' && !(*argv)[2]) { \ + ++argv, --argc; \ + break; \ + } \ + for (char *opt_ = &(*argv)[1], done_ = 0; !done_ && *opt_; ++opt_) { \ + switch (*opt_) + +#define SAMU_ARGEND \ + } \ + } + +#define SAMU_EARGF(x) \ + (done_ = 1, *++opt_ ? opt_ : argv[1] ? --argc, *++argv : ((x), abort(), (char *)0)) + +#endif diff --git a/include/external/samurai/build.h b/include/external/samurai/build.h new file mode 100644 index 000000000..0f13bdbf8 --- /dev/null +++ b/include/external/samurai/build.h @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_BUILD_H +#define MUON_EXTERNAL_SAMU_BUILD_H + +struct samu_node; + +/* reset state, so a new build can be executed */ +void samu_buildreset(struct samu_ctx *ctx); +/* schedule a particular target to be built */ +void samu_buildadd(struct samu_ctx *ctx, struct samu_node *n); +/* execute rules to build the scheduled targets */ +void samu_build(struct samu_ctx *ctx); + +#endif diff --git a/include/external/samurai/ctx.h b/include/external/samurai/ctx.h new file mode 100644 index 000000000..25a4bc5ed --- /dev/null +++ b/include/external/samurai/ctx.h @@ -0,0 +1,221 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_CTX_H +#define MUON_EXTERNAL_SAMU_CTX_H + +#include +#include +#include + +#include "platform/timer.h" + +struct samu_buffer { + char *data; + size_t len, cap; +}; + +struct samu_string { + size_t n; + char s[]; +}; + +/* an unevaluated string */ +struct samu_evalstring { + char *var; + struct samu_string *str; + struct samu_evalstring *next; +}; + +struct samu_hashtablekey { + uint64_t hash; + const char *str; + size_t len; +}; + +struct samu_buildoptions { + size_t maxjobs, maxfail; + _Bool verbose, explain, keepdepfile, keeprsp, dryrun; + const char *statusfmt; + double maxload; +}; + +struct samu_parseoptions { + _Bool dupbuildwarn; +}; + +enum samu_token { + SAMU_BUILD, + SAMU_DEFAULT, + SAMU_INCLUDE, + SAMU_POOL, + SAMU_RULE, + SAMU_SUBNINJA, + SAMU_VARIABLE, +}; + +struct samu_scanner { + FILE *f; + const char *path; + int chr, line, col; +}; + +struct samu_node { + /* shellpath is the escaped shell path, and is populated as needed by nodepath */ + struct samu_string *path, *shellpath; + + /* modification time of file (in nanoseconds) and build log entry (in seconds) */ + int64_t mtime, logmtime; + + /* generating edge and dependent edges */ + struct samu_edge *gen, **use; + size_t nuse; + + /* command hash used to build this output, read from build log */ + uint64_t hash; + + /* ID for .ninja_deps. -1 if not present in log. */ + int32_t id; + + /* does the node need to be rebuilt */ + _Bool dirty; +}; + +/* build rule, i.e., edge between inputs and outputs */ +struct samu_edge { + struct samu_rule *rule; + struct samu_pool *pool; + struct samu_environment *env; + + /* input and output nodes */ + struct samu_node **out, **in; + size_t nout, nin; + + /* index of first implicit output */ + size_t outimpidx; + /* index of first implicit and order-only input */ + size_t inimpidx, inorderidx; + + /* command hash */ + uint64_t hash; + + /* how many inputs need to be rebuilt or pruned before this edge is ready */ + size_t nblock; + /* how many inputs need to be pruned before all outputs can be pruned */ + size_t nprune; + + enum { + FLAG_WORK = 1 << 0, /* scheduled for build */ + FLAG_HASH = 1 << 1, /* calculated the command hash */ + FLAG_DIRTY_IN = 1 << 3, /* dirty input */ + FLAG_DIRTY_OUT = 1 << 4, /* missing or outdated output */ + FLAG_DIRTY = FLAG_DIRTY_IN | FLAG_DIRTY_OUT, + FLAG_CYCLE = 1 << 5, /* used for cycle detection */ + FLAG_DEPS = 1 << 6, /* dependencies loaded */ + } flags; + + /* used to coordinate ready work in build() */ + struct samu_edge *worknext; + /* used for alledges linked list */ + struct samu_edge *allnext; +}; + +struct samu_nodearray { + struct samu_node **node; + size_t len; +}; + +struct samu_entry { + struct samu_node *node; + struct samu_nodearray deps; + int64_t mtime; +}; + +struct samu_rule { + char *name; + struct samu_treenode *bindings; +}; + +struct samu_pool { + char *name; + int numjobs, maxjobs; + + /* a queue of ready edges blocked by the pool's capacity */ + struct samu_edge *work; +}; + +struct samu_build_ctx { + struct samu_edge *work; + size_t nstarted, nfinished, ntotal; + bool consoleused; + struct timer timer; +}; + +struct samu_deps_ctx { + FILE *depsfile; + struct samu_entry *entries; + size_t entrieslen, entriescap; + + struct samu_buffer buf; + struct samu_nodearray deps; + size_t depscap; +}; + +struct samu_env_ctx { + struct samu_environment *rootenv; + struct samu_treenode *pools; + struct samu_environment *allenvs; +}; + +struct samu_graph_ctx { + struct samu_hashtable *allnodes; + struct samu_edge *alledges; +}; + +struct samu_log_ctx { + FILE *logfile; +}; + +struct samu_parse_ctx { + struct samu_node **deftarg; + size_t ndeftarg; +}; + +struct samu_scan_ctx { + struct samu_evalstring **paths; + size_t npaths; + size_t paths_max; + struct samu_buffer buf; +}; + +struct samu_arena { + size_t blocks_len, i, allocd, filled; + char **blocks; +}; + +struct samu_ctx { + struct samu_buildoptions buildopts; + struct samu_parseoptions parseopts; + + struct samu_build_ctx build; + struct samu_deps_ctx deps; + struct samu_env_ctx env; + struct samu_graph_ctx graph; + struct samu_log_ctx log; + struct samu_parse_ctx parse; + struct samu_scan_ctx scan; + + const char *argv0; + struct samu_rule phonyrule; + struct samu_pool consolepool; + struct samu_arena arena; +}; + +struct samu_tool { + const char *name; + int (*run)(struct samu_ctx *, int, char *[]); +}; +#endif diff --git a/include/external/samurai/deps.h b/include/external/samurai/deps.h new file mode 100644 index 000000000..577aba479 --- /dev/null +++ b/include/external/samurai/deps.h @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_DEPS_H +#define MUON_EXTERNAL_SAMU_DEPS_H + +struct samu_edge; + +void samu_depsinit(struct samu_ctx *ctx, const char *builddir); +void samu_depsclose(struct samu_ctx *ctx); +void samu_depsload(struct samu_ctx *ctx, struct samu_edge *e); +void samu_depsrecord(struct samu_ctx *ctx, struct samu_edge *e); + +#endif diff --git a/include/external/samurai/env.h b/include/external/samurai/env.h new file mode 100644 index 000000000..4915e4e62 --- /dev/null +++ b/include/external/samurai/env.h @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_ENV_H +#define MUON_EXTERNAL_SAMU_ENV_H + +struct samu_edge; +struct samu_evalstring; +struct samu_string; + +void samu_envinit(struct samu_ctx *ctx); + +/* create a new environment with an optional parent */ +struct samu_environment *samu_mkenv(struct samu_ctx *ctx, struct samu_environment *); +/* search environment and its parents for a variable, returning the value or NULL if not found */ +struct samu_string *samu_envvar(struct samu_environment *, char *); +/* add to environment a variable and its value, replacing the old value if there is one */ +void samu_envaddvar(struct samu_ctx *ctx, struct samu_environment *env, char *var, struct samu_string *val); +/* evaluate an unevaluated string within an environment, returning the result */ +struct samu_string *samu_enveval(struct samu_ctx *ctx, struct samu_environment *, struct samu_evalstring *); +/* search an environment and its parents for a rule, returning the rule or NULL if not found */ +struct samu_rule *samu_envrule(struct samu_environment *env, char *name); +/* add a rule to an environment, or fail if the rule already exists */ +void samu_envaddrule(struct samu_ctx *ctx, struct samu_environment *env, struct samu_rule *r); + +/* create a new rule with the given name */ +struct samu_rule *samu_mkrule(struct samu_ctx *ctx, char *); +/* add to rule a variable and its value */ +void samu_ruleaddvar(struct samu_ctx *ctx, struct samu_rule *r, char *var, struct samu_evalstring *val); + +/* create a new pool with the given name */ +struct samu_pool *samu_mkpool(struct samu_ctx *ctx, char *name); +/* lookup a pool by name, or fail if it does not exist */ +struct samu_pool *samu_poolget(struct samu_ctx *ctx, char *name); + +/* evaluate and return an edge's variable, optionally shell-escaped */ +struct samu_string *samu_edgevar(struct samu_ctx *ctx, struct samu_edge *e, char *var, bool escape); + +#endif diff --git a/include/external/samurai/graph.h b/include/external/samurai/graph.h new file mode 100644 index 000000000..2214b3358 --- /dev/null +++ b/include/external/samurai/graph.h @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_GRAPH_H +#define MUON_EXTERNAL_SAMU_GRAPH_H + +#include /* for uint64_t */ + +struct samu_string; + +/* set in the tv_nsec field of a node's mtime */ +enum { + /* we haven't stat the file yet */ + SAMU_MTIME_UNKNOWN = -1, + /* the file does not exist */ + SAMU_MTIME_MISSING = -2, +}; + +void samu_graphinit(struct samu_ctx *ctx); + +/* create a new node or return existing node */ +struct samu_node *samu_mknode(struct samu_ctx *ctx, struct samu_string *path); +/* lookup a node by name; returns NULL if it does not exist */ +struct samu_node *samu_nodeget(struct samu_ctx *ctx, const char *path, size_t len); +/* update the mtime field of a node */ +void samu_nodestat(struct samu_node *); +/* get a node's path, possibly escaped for the shell */ +struct samu_string *samu_nodepath(struct samu_ctx *ctx, struct samu_node *n, bool escape); +/* record the usage of a node by an edge */ +void samu_nodeuse(struct samu_ctx *ctx, struct samu_node *n, struct samu_edge *e); + +/* create a new edge with the given parent environment */ +struct samu_edge *samu_mkedge(struct samu_ctx *ctx, struct samu_environment *parent); +/* compute the murmurhash64a of an edge command and store it in the hash field */ +void samu_edgehash(struct samu_ctx *ctx, struct samu_edge *e); +/* add dependencies from $depfile or .ninja_deps as implicit inputs */ +void samu_edgeadddeps(struct samu_ctx *ctx, struct samu_edge *e, struct samu_node **deps, size_t ndeps); + +#endif diff --git a/include/external/samurai/htab.h b/include/external/samurai/htab.h new file mode 100644 index 000000000..a02fe4679 --- /dev/null +++ b/include/external/samurai/htab.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_HTAB_H +#define MUON_EXTERNAL_SAMU_HTAB_H + +#include /* for uint64_t */ + +void samu_htabkey(struct samu_hashtablekey *, const char *, size_t); + +struct samu_hashtable *samu_mkhtab(struct samu_arena *a, size_t cap); +void samu_delhtab(struct samu_hashtable *, void(void *)); +void **samu_htabput(struct samu_arena *a, struct samu_hashtable *h, struct samu_hashtablekey *k); +void *samu_htabget(struct samu_hashtable *, struct samu_hashtablekey *); + +uint64_t samu_murmurhash64a(const void *, size_t); + +#endif diff --git a/include/external/samurai/log.h b/include/external/samurai/log.h new file mode 100644 index 000000000..917d29549 --- /dev/null +++ b/include/external/samurai/log.h @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_LOG_H +#define MUON_EXTERNAL_SAMU_LOG_H + +struct samu_node; + +void samu_loginit(struct samu_ctx *ctx, const char *); +void samu_logclose(struct samu_ctx *ctx); +void samu_logrecord(struct samu_ctx *ctx, struct samu_node *); + +#endif diff --git a/include/external/samurai/parse.h b/include/external/samurai/parse.h new file mode 100644 index 000000000..15080ed78 --- /dev/null +++ b/include/external/samurai/parse.h @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_PARSE_H +#define MUON_EXTERNAL_SAMU_PARSE_H + +struct samu_environment; +struct samu_node; + +void samu_parseinit(struct samu_ctx *ctx); +void samu_parse(struct samu_ctx *ctx, const char *name, struct samu_environment *env); + +/* supported ninja version */ +enum { + ninjamajor = 1, + ninjaminor = 9, +}; + +/* execute a function with all default nodes */ +void samu_defaultnodes(struct samu_ctx *ctx, void fn(struct samu_ctx *ctx, struct samu_node *)); + +#endif diff --git a/include/external/samurai/samu.h b/include/external/samurai/samu.h new file mode 100644 index 000000000..0999fe554 --- /dev/null +++ b/include/external/samurai/samu.h @@ -0,0 +1,9 @@ +/* + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: GPL-3.0-only + */ + +#ifndef MUON_EXTERNAL_SAMU_SAMU_H +#define MUON_EXTERNAL_SAMU_SAMU_H +int samu_main(int argc, char *argv[]); +#endif diff --git a/include/external/samurai/scan.h b/include/external/samurai/scan.h new file mode 100644 index 000000000..5da3e7f5e --- /dev/null +++ b/include/external/samurai/scan.h @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_SCAN_H +#define MUON_EXTERNAL_SAMU_SCAN_H + +void samu_scaninit(struct samu_scanner *, const char *); +void samu_scanclose(struct samu_scanner *); + +void samu_scanerror(struct samu_scanner *, const char *, ...); +int samu_scankeyword(struct samu_ctx *ctx, struct samu_scanner *s, char **var); +char *samu_scanname(struct samu_ctx *ctx, struct samu_scanner *s); +struct samu_evalstring *samu_scanstring(struct samu_ctx *ctx, struct samu_scanner *s, bool path); +void samu_scanpaths(struct samu_ctx *ctx, struct samu_scanner *s); +void samu_scanchar(struct samu_scanner *, int); +int samu_scanpipe(struct samu_scanner *, int); +_Bool samu_scanindent(struct samu_scanner *); +void samu_scannewline(struct samu_scanner *); + +#endif diff --git a/include/external/samurai/tool.h b/include/external/samurai/tool.h new file mode 100644 index 000000000..f97719171 --- /dev/null +++ b/include/external/samurai/tool.h @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_TOOL_H +#define MUON_EXTERNAL_SAMU_TOOL_H +const struct samu_tool *samu_toolget(const char *); +#endif diff --git a/include/external/samurai/tree.h b/include/external/samurai/tree.h new file mode 100644 index 000000000..ad6af9122 --- /dev/null +++ b/include/external/samurai/tree.h @@ -0,0 +1,21 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_TREE_H +#define MUON_EXTERNAL_SAMU_TREE_H +/* binary tree node, such that keys are sorted lexicographically for fast lookup */ +struct samu_treenode { + char *key; + void *value; + struct samu_treenode *child[2]; + int height; +}; + +/* search a binary tree for a key, return the key's value or NULL */ +struct samu_treenode *samu_treefind(struct samu_treenode *, const char *); +/* insert into a binary tree a key and a value, replace and return the old value if the key already exists */ +void *samu_treeinsert(struct samu_ctx *ctx, struct samu_treenode **rootp, char *key, void *value); +#endif diff --git a/include/external/samurai/util.h b/include/external/samurai/util.h new file mode 100644 index 000000000..0799e640c --- /dev/null +++ b/include/external/samurai/util.h @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#ifndef MUON_EXTERNAL_SAMU_UTIL_H +#define MUON_EXTERNAL_SAMU_UTIL_H +void samu_warn(const char *, ...); +void samu_fatal(const char *, ...); + +void samu_arena_init(struct samu_arena *a); +void samu_arena_destroy(struct samu_arena *a); +void *samu_arena_alloc(struct samu_arena *a, size_t size); +void *samu_arena_realloc(struct samu_arena *a, void *p, size_t old, size_t new); + +void *samu_xmalloc(struct samu_arena *a, size_t); +void *samu_xreallocarray(struct samu_arena *a, void *, size_t old, size_t new, size_t item_size); +char *samu_xmemdup(struct samu_arena *a, const char *, size_t); +int samu_xasprintf(struct samu_arena *a, char **, const char *, ...); + +/* append a byte to a buffer */ +void samu_bufadd(struct samu_arena *a, struct samu_buffer *buf, char c); + +/* allocates a new string with length n. n + 1 bytes are allocated for + * s, but not initialized. */ +struct samu_string *samu_mkstr(struct samu_arena *a, size_t n); + +/* delete an unevaluated string */ +void samu_delevalstr(void *); + +/* canonicalizes the given path by removing duplicate slashes, and + * folding '/.' and 'foo/..' */ +void samu_canonpath(struct samu_string *); +/* make a directory (or parent directory of a file) recursively */ +int samu_makedirs(struct samu_string *, _Bool); +/* write a new file with the given name and contents */ +int samu_writefile(const char *, struct samu_string *); +#endif diff --git a/src/amalgam.c b/src/amalgam.c index d7808f50f..ec7cb5245 100644 --- a/src/amalgam.c +++ b/src/amalgam.c @@ -9,6 +9,8 @@ #define __EXTENSIONS__ #endif +#define NO_GETLOADAVG + #include "args.c" #include "backend/backend.c" #include "backend/common_args.c" @@ -30,7 +32,19 @@ #include "external/bestline_null.c" #include "external/libarchive_null.c" #include "external/libcurl_null.c" -#include "external/samurai_null.c" +#include "external/samurai.c" +#include "external/samurai/build.c" +#include "external/samurai/deps.c" +#include "external/samurai/env.c" +#include "external/samurai/graph.c" +#include "external/samurai/htab.c" +#include "external/samurai/log.c" +#include "external/samurai/parse.c" +#include "external/samurai/samu.c" +#include "external/samurai/scan.c" +#include "external/samurai/tool.c" +#include "external/samurai/tree.c" +#include "external/samurai/util.c" #include "external/tinyjson_null.c" #include "formats/editorconfig.c" #include "formats/ini.c" diff --git a/src/backend/ninja.c b/src/backend/ninja.c index cf0949eac..ff7d2204c 100644 --- a/src/backend/ninja.c +++ b/src/backend/ninja.c @@ -275,17 +275,6 @@ ninja_write_all(struct workspace *wk) int ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture) { - // XXX since samu was designed to be an executable and not a library, - // lots of the resource management is left to the OS. For instance, - // there are several important globals that are assumed to be - // zero-initialized. Not to mention memory "leaks". This is all fine - // since almost zero effort has been put in to making samu into a true - // libsamu, however it means that calling the internal samu more than - // once is riddled with UB. Prevent that with this hacky static - // variable by falling back to executing an external ninja-compatible - // tool if the internal samu has already been invoked. - static bool internal_samu_has_been_called = false; - const char *argstr; uint32_t argstr_argc; @@ -310,9 +299,7 @@ ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture have_stdout_fileno = true; } - if (have_samurai && !internal_samu_has_been_called && have_stdout_fileno) { - internal_samu_has_been_called = true; - + if (have_samurai && have_stdout_fileno) { join_args_argstr(wk, &argstr, &argstr_argc, args); argc = argstr_to_argv(argstr, argstr_argc, "samu", &argv); diff --git a/src/external/meson.build b/src/external/meson.build index 4cf7deef8..7e44e7984 100644 --- a/src/external/meson.build +++ b/src/external/meson.build @@ -16,7 +16,6 @@ foreach d : [ 'default_options': ['pkgconf:tests=false', 'warning_level=1'], }, ], - ['samurai', {'fallback': ['samurai', 'libsamu_dep']}], ['bestline'], ] name = d[0] @@ -46,3 +45,25 @@ dep_sources += [files('tinyjson.c')] if get_option('static') and dep_dict['libcurl'] external_deps += dependency('libbrotlidec', static: true) endif + +if get_option('samurai').disabled() + dep_dict += {'samurai': false} + dep_sources += files('samurai_null.c') +else + dep_dict += {'samurai': true} + dep_sources += files( + 'samurai/build.c', + 'samurai/deps.c', + 'samurai/env.c', + 'samurai/graph.c', + 'samurai/htab.c', + 'samurai/log.c', + 'samurai/parse.c', + 'samurai/samu.c', + 'samurai/scan.c', + 'samurai/tool.c', + 'samurai/tree.c', + 'samurai/util.c', + 'samurai.c', + ) +endif diff --git a/src/external/samurai.c b/src/external/samurai.c index cef04529d..221df7853 100644 --- a/src/external/samurai.c +++ b/src/external/samurai.c @@ -5,10 +5,9 @@ #include "compat.h" -#include - #include "buf_size.h" #include "external/samurai.h" +#include "external/samurai/samu.h" #include "platform/filesystem.h" #include "platform/path.h" diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c new file mode 100644 index 000000000..b1ed754e8 --- /dev/null +++ b/src/external/samurai/build.c @@ -0,0 +1,624 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#ifndef NO_GETLOADAVG +#define _BSD_SOURCE /* for getloadavg */ +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/build.h" +#include "external/samurai/deps.h" +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/log.h" +#include "external/samurai/util.h" + +struct samu_job { + struct samu_string *cmd; + struct samu_edge *edge; + struct samu_buffer buf; + size_t next; + pid_t pid; + int fd; + bool failed; +}; + +void +samu_buildreset(struct samu_ctx *ctx) +{ + struct samu_edge *e; + + for (e = ctx->graph.alledges; e; e = e->allnext) + e->flags &= ~FLAG_WORK; +} + +/* returns whether n1 is newer than n2, or false if n1 is NULL */ +static bool +samu_isnewer(struct samu_node *n1, struct samu_node *n2) +{ + return n1 && n1->mtime > n2->mtime; +} + +/* returns whether this output node is dirty in relation to the newest input */ +static bool +samu_isdirty(struct samu_ctx *ctx, struct samu_node *n, struct samu_node *newest, bool generator, bool restat) +{ + struct samu_edge *e; + + e = n->gen; + if (e->rule == &ctx->phonyrule) { + if (e->nin > 0 || n->mtime != SAMU_MTIME_MISSING) + return false; + if (ctx->buildopts.explain) + samu_warn("explain %s: phony and no inputs", n->path->s); + return true; + } + if (n->mtime == SAMU_MTIME_MISSING) { + if (ctx->buildopts.explain) + samu_warn("explain %s: missing", n->path->s); + return true; + } + if (samu_isnewer(newest, n) && (!restat || n->logmtime == SAMU_MTIME_MISSING)) { + if (ctx->buildopts.explain) { + samu_warn("explain %s: older than input '%s': %" PRId64 " vs %" PRId64, + n->path->s, newest->path->s, n->mtime, newest->mtime); + } + return true; + } + if (n->logmtime == SAMU_MTIME_MISSING) { + if (!generator) { + if (ctx->buildopts.explain) + samu_warn("explain %s: no record in .ninja_log", n->path->s); + return true; + } + } else if (newest && n->logmtime < newest->mtime) { + if (ctx->buildopts.explain) { + samu_warn("explain %s: recorded mtime is older than input '%s': %" PRId64 " vs %" PRId64, + n->path->s, newest->path->s, n->logmtime, newest->mtime); + } + return true; + } + if (generator) + return false; + samu_edgehash(ctx, e); + if (e->hash == n->hash) + return false; + if (ctx->buildopts.explain) + samu_warn("explain %s: command line changed", n->path->s); + return true; +} + +/* add an edge to the work queue */ +static void +samu_queue(struct samu_ctx *ctx, struct samu_edge *e) +{ + struct samu_edge **front = &ctx->build.work; + + if (e->pool && e->rule != &ctx->phonyrule) { + if (e->pool->numjobs == e->pool->maxjobs) + front = &e->pool->work; + else + ++e->pool->numjobs; + } + e->worknext = *front; + *front = e; +} + +void +samu_buildadd(struct samu_ctx *ctx, struct samu_node *n) +{ + struct samu_edge *e; + struct samu_node *newest; + size_t i; + bool generator, restat; + + e = n->gen; + if (!e) { + if (n->mtime == SAMU_MTIME_UNKNOWN) + samu_nodestat(n); + if (n->mtime == SAMU_MTIME_MISSING) + samu_fatal("file is missing and not created by any action: '%s'", n->path->s); + n->dirty = false; + return; + } + if (e->flags & FLAG_CYCLE) + samu_fatal("dependency cycle involving '%s'", n->path->s); + if (e->flags & FLAG_WORK) + return; + e->flags |= FLAG_CYCLE | FLAG_WORK; + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + n->dirty = false; + if (n->mtime == SAMU_MTIME_UNKNOWN) + samu_nodestat(n); + } + samu_depsload(ctx, e); + e->nblock = 0; + newest = NULL; + for (i = 0; i < e->nin; ++i) { + n = e->in[i]; + samu_buildadd(ctx, n); + if (i < e->inorderidx) { + if (n->dirty) + e->flags |= FLAG_DIRTY_IN; + if (n->mtime != SAMU_MTIME_MISSING && !samu_isnewer(newest, n)) + newest = n; + } + if (n->dirty || (n->gen && n->gen->nblock > 0)) + ++e->nblock; + } + /* all outputs are dirty if any are older than the newest input */ + generator = samu_edgevar(ctx, e, "generator", true); + restat = samu_edgevar(ctx, e, "restat", true); + for (i = 0; i < e->nout && !(e->flags & FLAG_DIRTY_OUT); ++i) { + n = e->out[i]; + if (samu_isdirty(ctx, n, newest, generator, restat)) { + n->dirty = true; + e->flags |= FLAG_DIRTY_OUT; + } + } + if (e->flags & FLAG_DIRTY) { + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + if (ctx->buildopts.explain && !n->dirty) { + if (e->flags & FLAG_DIRTY_IN) + samu_warn("explain %s: input is dirty", n->path->s); + else if (e->flags & FLAG_DIRTY_OUT) + samu_warn("explain %s: output of generating action is dirty", n->path->s); + } + n->dirty = true; + } + } + if (!(e->flags & FLAG_DIRTY_OUT)) + e->nprune = e->nblock; + if (e->flags & FLAG_DIRTY) { + if (e->nblock == 0) + samu_queue(ctx, e); + if (e->rule != &ctx->phonyrule) + ++ctx->build.ntotal; + } + e->flags &= ~FLAG_CYCLE; +} + +static size_t +samu_formatstatus(struct samu_ctx *ctx, char *buf, size_t len) +{ + const char *fmt; + size_t ret = 0; + int n; + + for (fmt = ctx->buildopts.statusfmt; *fmt; ++fmt) { + if (*fmt != '%' || *++fmt == '%') { + if (len > 1) { + *buf++ = *fmt; + --len; + } + ++ret; + continue; + } + n = 0; + switch (*fmt) { + case 's': + n = snprintf(buf, len, "%zu", ctx->build.nstarted); + break; + case 'f': + n = snprintf(buf, len, "%zu", ctx->build.nfinished); + break; + case 't': + n = snprintf(buf, len, "%zu", ctx->build.ntotal); + break; + case 'r': + n = snprintf(buf, len, "%zu", ctx->build.nstarted - ctx->build.nfinished); + break; + case 'u': + n = snprintf(buf, len, "%zu", ctx->build.ntotal - ctx->build.nstarted); + break; + case 'p': + n = snprintf(buf, len, "%3zu%%", 100 * ctx->build.nfinished / ctx->build.ntotal); + break; + case 'o': + n = snprintf(buf, len, "%.1f", ctx->build.nfinished / timer_read(&ctx->build.timer)); + break; + case 'e': + n = snprintf(buf, len, "%.3f", timer_read(&ctx->build.timer)); + break; + default: + samu_fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *fmt); + continue; /* unreachable, but avoids warning */ + } + if (n < 0) + samu_fatal("snprintf:"); + ret += n; + if ((size_t)n > len) + n = len; + buf += n; + len -= n; + } + if (len > 0) + *buf = '\0'; + return ret; +} + +static void +samu_printstatus(struct samu_ctx *ctx, struct samu_edge *e, struct samu_string *cmd) +{ + struct samu_string *description; + char status[256]; + + description = ctx->buildopts.verbose ? NULL : samu_edgevar(ctx, e, "description", true); + if (!description || description->n == 0) + description = cmd; + samu_formatstatus(ctx, status, sizeof(status)); + fputs(status, stdout); + puts(description->s); +} + +static int +samu_jobstart(struct samu_ctx *ctx, struct samu_job *j, struct samu_edge *e) +{ + extern char **environ; + size_t i; + struct samu_node *n; + struct samu_string *rspfile, *content; + int fd[2]; + posix_spawn_file_actions_t actions; + char *argv[] = {"/bin/sh", "-c", NULL, NULL}; + + ++ctx->build.nstarted; + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + if (n->mtime == SAMU_MTIME_MISSING) { + if (samu_makedirs(n->path, true) < 0) + goto err0; + } + } + rspfile = samu_edgevar(ctx, e, "rspfile", false); + if (rspfile) { + content = samu_edgevar(ctx, e, "rspfile_content", true); + if (samu_writefile(rspfile->s, content) < 0) + goto err0; + } + + if (pipe(fd) < 0) { + samu_warn("pipe:"); + goto err1; + } + j->edge = e; + j->cmd = samu_edgevar(ctx, e, "command", true); + j->fd = fd[0]; + argv[2] = j->cmd->s; + + if (!ctx->build.consoleused) + samu_printstatus(ctx, e, j->cmd); + + if ((errno = posix_spawn_file_actions_init(&actions))) { + samu_warn("posix_spawn_file_actions_init:"); + goto err2; + } + if ((errno = posix_spawn_file_actions_addclose(&actions, fd[0]))) { + samu_warn("posix_spawn_file_actions_addclose:"); + goto err3; + } + if (e->pool != &ctx->consolepool) { + if ((errno = posix_spawn_file_actions_addopen(&actions, 0, "/dev/null", O_RDONLY, 0))) { + samu_warn("posix_spawn_file_actions_addopen:"); + goto err3; + } + if ((errno = posix_spawn_file_actions_adddup2(&actions, fd[1], 1))) { + samu_warn("posix_spawn_file_actions_adddup2:"); + goto err3; + } + if ((errno = posix_spawn_file_actions_adddup2(&actions, fd[1], 2))) { + samu_warn("posix_spawn_file_actions_adddup2:"); + goto err3; + } + if ((errno = posix_spawn_file_actions_addclose(&actions, fd[1]))) { + samu_warn("posix_spawn_file_actions_addclose:"); + goto err3; + } + } + if ((errno = posix_spawn(&j->pid, argv[0], &actions, NULL, argv, environ))) { + samu_warn("posix_spawn %s:", j->cmd->s); + goto err3; + } + posix_spawn_file_actions_destroy(&actions); + close(fd[1]); + j->failed = false; + if (e->pool == &ctx->consolepool) + ctx->build.consoleused = true; + + return j->fd; + +err3: + posix_spawn_file_actions_destroy(&actions); +err2: + close(fd[0]); + close(fd[1]); +err1: + if (rspfile && !ctx->buildopts.keeprsp) + remove(rspfile->s); +err0: + return -1; +} + +static void +samu_nodedone(struct samu_ctx *ctx, struct samu_node *n, bool prune) +{ + struct samu_edge *e; + size_t i, j; + + for (i = 0; i < n->nuse; ++i) { + e = n->use[i]; + /* skip edges not used in this build */ + if (!(e->flags & FLAG_WORK)) + continue; + if (!(e->flags & (prune ? FLAG_DIRTY_OUT : FLAG_DIRTY)) && --e->nprune == 0) { + /* either edge was clean (possible with order-only + * inputs), or all its blocking inputs were pruned, so + * its outputs can be pruned as well */ + for (j = 0; j < e->nout; ++j) + samu_nodedone(ctx, e->out[j], true); + if (e->flags & FLAG_DIRTY && e->rule != &ctx->phonyrule) + --ctx->build.ntotal; + } else if (--e->nblock == 0) { + samu_queue(ctx, e); + } + } +} + +static bool +samu_shouldprune(struct samu_edge *e, struct samu_node *n, int64_t old) +{ + struct samu_node *in, *newest; + size_t i; + + if (old != n->mtime) + return false; + newest = NULL; + for (i = 0; i < e->inorderidx; ++i) { + in = e->in[i]; + samu_nodestat(in); + if (in->mtime != SAMU_MTIME_MISSING && !samu_isnewer(newest, in)) + newest = in; + } + if (newest) + n->logmtime = newest->mtime; + + return true; +} + +static void +samu_edgedone(struct samu_ctx *ctx, struct samu_edge *e) +{ + struct samu_node *n; + size_t i; + struct samu_string *rspfile; + bool restat; + int64_t old; + + restat = samu_edgevar(ctx, e, "restat", true); + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + old = n->mtime; + samu_nodestat(n); + n->logmtime = n->mtime == SAMU_MTIME_MISSING ? 0 : n->mtime; + samu_nodedone(ctx, n, restat && samu_shouldprune(e, n, old)); + } + rspfile = samu_edgevar(ctx, e, "rspfile", false); + if (rspfile && !ctx->buildopts.keeprsp) + remove(rspfile->s); + samu_edgehash(ctx, e); + samu_depsrecord(ctx, e); + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + n->hash = e->hash; + samu_logrecord(ctx, n); + } +} + +static void +samu_jobdone(struct samu_ctx *ctx, struct samu_job *j) +{ + int status; + struct samu_edge *e, *new; + struct samu_pool *p; + + ++ctx->build.nfinished; + if (waitpid(j->pid, &status, 0) < 0) { + samu_warn("waitpid %d:", j->pid); + j->failed = true; + } else if (WIFEXITED(status)) { + if (WEXITSTATUS(status) != 0) { + samu_warn("job failed: %s", j->cmd->s); + j->failed = true; + } + } else if (WIFSIGNALED(status)) { + samu_warn("job terminated due to signal %d: %s", WTERMSIG(status), j->cmd->s); + j->failed = true; + } else { + /* cannot happen according to POSIX */ + samu_warn("job status unknown: %s", j->cmd->s); + j->failed = true; + } + close(j->fd); + if (j->buf.len && (!ctx->build.consoleused || j->failed)) + fwrite(j->buf.data, 1, j->buf.len, stdout); + j->buf.len = 0; + e = j->edge; + if (e->pool) { + p = e->pool; + + if (p == &ctx->consolepool) + ctx->build.consoleused = false; + /* move edge from pool queue to main work queue */ + if (p->work) { + new = p->work; + p->work = p->work->worknext; + new->worknext = ctx->build.work; + ctx->build.work = new; + } else { + --p->numjobs; + } + } + if (!j->failed) + samu_edgedone(ctx, e); +} + +/* returns whether a job still has work to do. if not, sets j->failed */ +static bool +samu_jobwork(struct samu_ctx *ctx, struct samu_job *j) +{ + char *newdata; + size_t newcap; + ssize_t n; + + if (j->buf.cap - j->buf.len < BUFSIZ / 2) { + newcap = j->buf.cap + BUFSIZ; + newdata = samu_arena_realloc(&ctx->arena, j->buf.data, j->buf.cap, newcap); + if (!newdata) { + samu_warn("realloc:"); + goto kill; + } + j->buf.cap = newcap; + j->buf.data = newdata; + } + n = read(j->fd, j->buf.data + j->buf.len, j->buf.cap - j->buf.len); + if (n > 0) { + j->buf.len += n; + return true; + } + if (n == 0) + goto done; + samu_warn("read:"); + +kill: + kill(j->pid, SIGTERM); + j->failed = true; +done: + samu_jobdone(ctx, j); + + return false; +} + +/* queries the system load average */ +static double +samu_queryload(void) +{ +#ifdef NO_GETLOADAVG + return 0; +#else + double load; + + if (getloadavg(&load, 1) == -1) { + samu_warn("getloadavg:"); + load = 100.0; + } + + return load; +#endif +} + +void +samu_build(struct samu_ctx *ctx) +{ + struct samu_job *jobs = NULL; + struct pollfd *fds = NULL; + size_t i, next = 0, jobslen = 0, maxjobs = ctx->buildopts.maxjobs, numjobs = 0, numfail = 0; + struct samu_edge *e; + + if (ctx->build.ntotal == 0) { + return; + } + + timer_start(&ctx->build.timer); + samu_formatstatus(ctx, NULL, 0); + + ctx->build.nstarted = 0; + for (;;) { + /* limit number of of jobs based on load */ + if (ctx->buildopts.maxload) + maxjobs = samu_queryload() > ctx->buildopts.maxload ? 1 : ctx->buildopts.maxjobs; + /* start ready edges */ + while (ctx->build.work && numjobs < maxjobs && numfail < ctx->buildopts.maxfail) { + e = ctx->build.work; + ctx->build.work = ctx->build.work->worknext; + if (e->rule != &ctx->phonyrule && ctx->buildopts.dryrun) { + ++ctx->build.nstarted; + samu_printstatus(ctx, e, samu_edgevar(ctx, e, "command", true)); + ++ctx->build.nfinished; + } + if (e->rule == &ctx->phonyrule || ctx->buildopts.dryrun) { + for (i = 0; i < e->nout; ++i) + samu_nodedone(ctx, e->out[i], false); + continue; + } + if (next == jobslen) { + size_t newjobslen; + newjobslen = jobslen ? jobslen * 2 : 8; + if (newjobslen > ctx->buildopts.maxjobs) + newjobslen = ctx->buildopts.maxjobs; + jobs = samu_xreallocarray(&ctx->arena, jobs, jobslen, newjobslen, sizeof(jobs[0])); + fds = samu_xreallocarray(&ctx->arena, fds, jobslen, newjobslen, sizeof(fds[0])); + jobslen = newjobslen; + for (i = next; i < jobslen; ++i) { + jobs[i].buf.data = NULL; + jobs[i].buf.len = 0; + jobs[i].buf.cap = 0; + jobs[i].next = i + 1; + fds[i].fd = -1; + fds[i].events = POLLIN; + } + } + fds[next].fd = samu_jobstart(ctx, &jobs[next], e); + if (fds[next].fd < 0) { + samu_warn("job failed to start"); + ++numfail; + } else { + next = jobs[next].next; + ++numjobs; + } + } + if (numjobs == 0) + break; + if (poll(fds, jobslen, 5000) < 0) + samu_fatal("poll:"); + for (i = 0; i < jobslen; ++i) { + if (!fds[i].revents || samu_jobwork(ctx, &jobs[i])) + continue; + --numjobs; + jobs[i].next = next; + fds[i].fd = -1; + next = i; + if (jobs[i].failed) + ++numfail; + } + } + if (numfail > 0) { + if (numfail < ctx->buildopts.maxfail) + samu_fatal("cannot make progress due to previous errors"); + else if (numfail > 1) + samu_fatal("subcommands failed"); + else + samu_fatal("subcommand failed"); + } + ctx->build.ntotal = 0; /* reset in case we just rebuilt the manifest */ +} diff --git a/src/external/samurai/deps.c b/src/external/samurai/deps.c new file mode 100644 index 000000000..1207802a0 --- /dev/null +++ b/src/external/samurai/deps.c @@ -0,0 +1,500 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/build.h" +#include "external/samurai/deps.h" +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/util.h" + +/* +.ninja_deps file format + +The header identifying the format is the string "# ninjadeps\n", followed by a +4-byte integer specifying the format version. After this is a series of binary +records. All integers in .ninja_deps are written in system byte-order. + +A record starts with a 4-byte integer indicating the record type and size. If +the high bit is set, then it is a dependency record. Otherwise, it is a node +record. In either case, the remaining 31 bits specify the size in bytes of the +rest of the record. The size must be a multiple of 4, and no larger than than +2^19. + +Node records are given in incrementing ID order, and must be given before any +dependency record that refers to it. The last 4-byte integer in the record is +used as a checksum to prevent corruption. Counting from 0, the n-th node record +(specifying the node with ID n) will have a checksum of ~n (bitwise negation of +n). The remaining bytes of the record specify the path of the node, padded with +NUL bytes to the next 4-byte boundary (start of the checksum value). + +A dependency record contains a list of dependencies for the edge that built a +particular node. The first 4-byte integer is the node ID. The second and third +4-byte integers are the low and high 32-bits of the UNIX mtime (in nanoseconds) +of the node when it was built. Following this is a sequence of 4-byte integers +specifying the IDs of the dependency nodes for this edge, which will have been +specified previously in node records. +*/ + +/* maximum record size (in bytes) */ +#define SAMU_MAX_RECORD_SIZE (1 << 19) + +static const char ninja_depsname[] = ".ninja_deps"; +static const char ninja_depstmpname[] = ".ninja_deps.tmp"; +static const char ninja_depsheader[] = "# ninjadeps\n"; +static const uint32_t ninja_depsver = 4; + +static void +samu_depswrite(struct samu_ctx *ctx, const void *p, size_t n, size_t m) +{ + if (fwrite(p, n, m, ctx->deps.depsfile) != m) + samu_fatal("deps log write:"); +} + +static bool +samu_recordid(struct samu_ctx *ctx, struct samu_node *n) +{ + uint32_t sz, chk; + + if (n->id != -1) + return false; + if (ctx->deps.entrieslen == INT32_MAX) + samu_fatal("too many nodes"); + n->id = ctx->deps.entrieslen++; + sz = (n->path->n + 7) & ~3; + if (sz + 4 >= SAMU_MAX_RECORD_SIZE) + samu_fatal("ID record too large"); + samu_depswrite(ctx, &sz, 4, 1); + samu_depswrite(ctx, n->path->s, 1, n->path->n); + samu_depswrite(ctx, (char[4]){0}, 1, sz - n->path->n - 4); + chk = ~n->id; + samu_depswrite(ctx, &chk, 4, 1); + + return true; +} + +static void +samu_recorddeps(struct samu_ctx *ctx, struct samu_node *out, struct samu_nodearray *deps, int64_t mtime) +{ + uint32_t sz, m; + size_t i; + + sz = 12 + deps->len * 4; + if (sz + 4 >= SAMU_MAX_RECORD_SIZE) + samu_fatal("deps record too large"); + sz |= 0x80000000; + samu_depswrite(ctx, &sz, 4, 1); + samu_depswrite(ctx, &out->id, 4, 1); + m = mtime & 0xffffffff; + samu_depswrite(ctx, &m, 4, 1); + m = (mtime >> 32) & 0xffffffff; + samu_depswrite(ctx, &m, 4, 1); + for (i = 0; i < deps->len; ++i) + samu_depswrite(ctx, &deps->node[i]->id, 4, 1); +} + +void +samu_depsinit(struct samu_ctx *ctx, const char *builddir) +{ + char *depspath = (char *)ninja_depsname, *depstmppath = (char *)ninja_depstmpname; + uint32_t *buf, cap, ver, sz, id; + size_t len, i, j, nrecord; + bool isdep; + struct samu_string *path; + struct samu_node *n; + struct samu_edge *e; + struct samu_entry *entry, *oldentries; + + /* XXX: when ninja hits a bad record, it truncates the log to the last + * good record. perhaps we should do the same. */ + + if (ctx->deps.depsfile) { + fclose(ctx->deps.depsfile); + ctx->deps.depsfile = NULL; + } + ctx->deps.entrieslen = 0; + cap = BUFSIZ; + buf = samu_xmalloc(&ctx->arena, cap); + if (builddir) + samu_xasprintf(&ctx->arena, &depspath, "%s/%s", builddir, ninja_depsname); + ctx->deps.depsfile = fopen(depspath, "r+"); + if (!ctx->deps.depsfile) { + if (errno != ENOENT) + samu_fatal("open %s:", depspath); + goto rewrite; + } + if (!fgets((char *)buf, sizeof(ninja_depsheader), ctx->deps.depsfile)) + goto rewrite; + if (strcmp((char *)buf, ninja_depsheader) != 0) { + samu_warn("invalid deps log header"); + goto rewrite; + } + if (fread(&ver, sizeof(ver), 1, ctx->deps.depsfile) != 1) { + samu_warn(ferror(ctx->deps.depsfile) ? "deps log read:" : "deps log truncated"); + goto rewrite; + } + if (ver != ninja_depsver) { + samu_warn("unknown deps log version"); + goto rewrite; + } + for (nrecord = 0;; ++nrecord) { + if (fread(&sz, sizeof(sz), 1, ctx->deps.depsfile) != 1) + break; + isdep = sz & 0x80000000; + sz &= 0x7fffffff; + if (sz > SAMU_MAX_RECORD_SIZE) { + samu_warn("deps record too large"); + goto rewrite; + } + if (sz > cap) { + do cap *= 2; + while (sz > cap); + buf = samu_xmalloc(&ctx->arena, cap); + } + if (fread(buf, sz, 1, ctx->deps.depsfile) != 1) { + samu_warn(ferror(ctx->deps.depsfile) ? "deps log read:" : "deps log truncated"); + goto rewrite; + } + if (sz % 4) { + samu_warn("invalid size, must be multiple of 4: %" PRIu32, sz); + goto rewrite; + } + if (isdep) { + if (sz < 12) { + samu_warn("invalid size, must be at least 12: %" PRIu32, sz); + goto rewrite; + } + sz -= 12; + id = buf[0]; + if (id >= ctx->deps.entrieslen) { + samu_warn("invalid node ID: %" PRIu32, id); + goto rewrite; + } + entry = &ctx->deps.entries[id]; + entry->mtime = (int64_t)buf[2] << 32 | buf[1]; + e = entry->node->gen; + if (!e || !samu_edgevar(ctx, e, "deps", true)) + continue; + sz /= 4; + entry->deps.len = sz; + entry->deps.node = samu_xreallocarray(&ctx->arena, NULL, 0, sz, sizeof(n)); + for (i = 0; i < sz; ++i) { + id = buf[3 + i]; + if (id >= ctx->deps.entrieslen) { + samu_warn("invalid node ID: %" PRIu32, id); + goto rewrite; + } + entry->deps.node[i] = ctx->deps.entries[id].node; + } + } else { + if (sz <= 4) { + samu_warn("invalid size, must be greater than 4: %" PRIu32, sz); + goto rewrite; + } + if (ctx->deps.entrieslen != ~buf[sz / 4 - 1]) { + samu_warn("corrupt deps log, bad checksum"); + goto rewrite; + } + if (ctx->deps.entrieslen == INT32_MAX) { + samu_warn("too many nodes in deps log"); + goto rewrite; + } + len = sz - 4; + while (((char *)buf)[len - 1] == '\0') + --len; + path = samu_mkstr(&ctx->arena, len); + memcpy(path->s, buf, len); + path->s[len] = '\0'; + + n = samu_mknode(ctx, path); + if (ctx->deps.entrieslen >= ctx->deps.entriescap) { + size_t newcap = ctx->deps.entriescap ? ctx->deps.entriescap * 2 : 1024; + ctx->deps.entries = samu_xreallocarray(&ctx->arena, ctx->deps.entries, ctx->deps.entriescap, newcap, sizeof(ctx->deps.entries[0])); + ctx->deps.entriescap = newcap; + } + n->id = ctx->deps.entrieslen; + ctx->deps.entries[ctx->deps.entrieslen++] = (struct samu_entry){.node = n}; + } + } + if (ferror(ctx->deps.depsfile)) { + samu_warn("deps log read:"); + goto rewrite; + } + if (nrecord <= 1000 || nrecord < 3 * ctx->deps.entrieslen) { + return; + } + +rewrite: + if (ctx->deps.depsfile) { + fclose(ctx->deps.depsfile); + ctx->deps.depsfile = NULL; + } + if (builddir) + samu_xasprintf(&ctx->arena, &depstmppath, "%s/%s", builddir, ninja_depstmpname); + ctx->deps.depsfile = fopen(depstmppath, "w"); + if (!ctx->deps.depsfile) + samu_fatal("open %s:", depstmppath); + samu_depswrite(ctx, ninja_depsheader, 1, sizeof(ninja_depsheader) - 1); + samu_depswrite(ctx, &ninja_depsver, 1, sizeof(ninja_depsver)); + + /* reset ID for all current entries */ + for (i = 0; i < ctx->deps.entrieslen; ++i) + ctx->deps.entries[i].node->id = -1; + /* save a temporary copy of the old entries */ + oldentries = samu_xreallocarray(&ctx->arena, NULL, 0, ctx->deps.entrieslen, sizeof(ctx->deps.entries[0])); + memcpy(oldentries, ctx->deps.entries, ctx->deps.entrieslen * sizeof(ctx->deps.entries[0])); + + len = ctx->deps.entrieslen; + ctx->deps.entrieslen = 0; + for (i = 0; i < len; ++i) { + entry = &oldentries[i]; + if (!entry->deps.len) + continue; + samu_recordid(ctx, entry->node); + ctx->deps.entries[entry->node->id] = *entry; + for (j = 0; j < entry->deps.len; ++j) + samu_recordid(ctx, entry->deps.node[j]); + samu_recorddeps(ctx, entry->node, &entry->deps, entry->mtime); + } + fflush(ctx->deps.depsfile); + if (ferror(ctx->deps.depsfile)) + samu_fatal("deps log write failed"); + if (rename(depstmppath, depspath) < 0) + samu_fatal("deps log rename:"); +} + +void +samu_depsclose(struct samu_ctx *ctx) +{ + fflush(ctx->deps.depsfile); + if (ferror(ctx->deps.depsfile)) + samu_fatal("deps log write failed"); + fclose(ctx->deps.depsfile); + ctx->deps.depsfile = NULL; +} + +static struct samu_nodearray * +samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) +{ + struct samu_string *in, *out = NULL; + FILE *f; + int c, n; + bool sawcolon; + + ctx->deps.deps.len = 0; + f = fopen(name, "r"); + if (!f) { + if (errno == ENOENT && allowmissing) + return &ctx->deps.deps; + return NULL; + } + sawcolon = false; + ctx->deps.buf.len = 0; + c = getc(f); + for (;;) { + /* TODO: this parser needs to be rewritten to be made simpler */ + while (isalnum(c) || strchr("$+,-./@\\_", c)) { + switch (c) { + case '\\': + /* handle the crazy escaping generated by clang and gcc */ + n = 0; + do { + c = getc(f); + if (++n % 2 == 0) + samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + } while (c == '\\'); + switch (c) { + case '#': + /* assume no comments */ + for (; n > 2; n -= 2) + samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + break; + case ' ': + case '\t': + if (n % 2 != 0) + break; + /* fallthrough */ + default: + for (; n > 0; n -= 2) + samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + continue; + } + break; + case '$': + c = getc(f); + if (c != '$') { + samu_warn("bad depfile: contains variable reference"); + goto err; + } + break; + } + samu_bufadd(&ctx->arena, &ctx->deps.buf, c); + c = getc(f); + } + if (sawcolon) { + if (!isspace(c) && c != EOF) { + samu_warn("bad depfile: '%c' is not a valid target character", c); + goto err; + } + if (ctx->deps.buf.len > 0) { + if (ctx->deps.deps.len == ctx->deps.depscap) { + size_t newcap = ctx->deps.deps.node ? ctx->deps.depscap * 2 : 32; + ctx->deps.deps.node = samu_xreallocarray(&ctx->arena, ctx->deps.deps.node, ctx->deps.depscap, newcap, sizeof(ctx->deps.deps.node[0])); + ctx->deps.depscap = newcap; + } + in = samu_mkstr(&ctx->arena, ctx->deps.buf.len); + memcpy(in->s, ctx->deps.buf.data, ctx->deps.buf.len); + in->s[ctx->deps.buf.len] = '\0'; + ctx->deps.deps.node[ctx->deps.deps.len++] = samu_mknode(ctx, in); + } + if (c == '\n') { + sawcolon = false; + do c = getc(f); + while (c == '\n'); + } + if (c == EOF) + break; + } else { + while (isblank(c)) + c = getc(f); + if (c == EOF) + break; + if (c != ':') { + samu_warn("bad depfile: expected ':', saw '%c'", c); + goto err; + } + if (!out) { + out = samu_mkstr(&ctx->arena, ctx->deps.buf.len); + memcpy(out->s, ctx->deps.buf.data, ctx->deps.buf.len); + out->s[ctx->deps.buf.len] = '\0'; + } else if (out->n != ctx->deps.buf.len || memcmp(ctx->deps.buf.data, out->s, ctx->deps.buf.len) != 0) { + samu_warn("bad depfile: multiple outputs: %.*s != %s", (int)ctx->deps.buf.len, ctx->deps.buf.data, out->s); + goto err; + } + sawcolon = true; + c = getc(f); + } + ctx->deps.buf.len = 0; + for (;;) { + if (c == '\\') { + if (getc(f) != '\n') { + samu_warn("bad depfile: '\\' only allowed before newline"); + goto err; + } + } else if (!isblank(c)) { + break; + } + c = getc(f); + } + } + if (ferror(f)) { + samu_warn("depfile read:"); + goto err; + } + fclose(f); + return &ctx->deps.deps; + +err: + fclose(f); + return NULL; +} + +void +samu_depsload(struct samu_ctx *ctx, struct samu_edge *e) +{ + struct samu_string *deptype, *depfile; + struct samu_nodearray *deps = NULL; + struct samu_node *n; + + if (e->flags & FLAG_DEPS) + return; + e->flags |= FLAG_DEPS; + n = e->out[0]; + deptype = samu_edgevar(ctx, e, "deps", true); + if (deptype) { + if (n->id != -1 && n->mtime <= ctx->deps.entries[n->id].mtime) + deps = &ctx->deps.entries[n->id].deps; + else if (ctx->buildopts.explain) + samu_warn("explain %s: missing or outdated record in .ninja_deps", n->path->s); + } else { + depfile = samu_edgevar(ctx, e, "depfile", false); + if (!depfile) + return; + deps = samu_depsparse(ctx, depfile->s, false); + if (ctx->buildopts.explain && !deps) + samu_warn("explain %s: missing or invalid depfile", n->path->s); + } + if (deps) { + samu_edgeadddeps(ctx, e, deps->node, deps->len); + } else { + n->dirty = true; + e->flags |= FLAG_DIRTY_OUT; + } +} + +void +samu_depsrecord(struct samu_ctx *ctx, struct samu_edge *e) +{ + struct samu_string *deptype, *depfile; + struct samu_nodearray *deps; + struct samu_node *out, *n; + struct samu_entry *entry; + size_t i; + bool update; + + deptype = samu_edgevar(ctx, e, "deps", true); + if (!deptype || deptype->n == 0) + return; + if (strcmp(deptype->s, "gcc") != 0) { + samu_warn("unsuported deps type: %s", deptype->s); + return; + } + depfile = samu_edgevar(ctx, e, "depfile", false); + if (!depfile || depfile->n == 0) { + samu_warn("deps but no depfile"); + return; + } + out = e->out[0]; + deps = samu_depsparse(ctx, depfile->s, true); + if (!ctx->buildopts.keepdepfile) + remove(depfile->s); + if (!deps) + return; + update = false; + entry = NULL; + if (samu_recordid(ctx, out)) { + update = true; + } else { + entry = &ctx->deps.entries[out->id]; + if (entry->mtime != out->mtime || entry->deps.len != deps->len) + update = true; + for (i = 0; i < deps->len && !update; ++i) { + if (entry->deps.node[i] != deps->node[i]) + update = true; + } + } + for (i = 0; i < deps->len; ++i) { + n = deps->node[i]; + if (samu_recordid(ctx, n)) + update = true; + } + if (update) { + samu_recorddeps(ctx, out, deps, out->mtime); + if (fflush(ctx->deps.depsfile) < 0) + samu_fatal("deps log flush:"); + } +} diff --git a/src/external/samurai/env.c b/src/external/samurai/env.c new file mode 100644 index 000000000..712e5f393 --- /dev/null +++ b/src/external/samurai/env.c @@ -0,0 +1,261 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/tree.h" +#include "external/samurai/util.h" + +struct samu_environment { + struct samu_environment *parent; + struct samu_treenode *bindings; + struct samu_treenode *rules; + struct samu_environment *allnext; +}; + +static void samu_addpool(struct samu_ctx *ctx, struct samu_pool *p); + +void +samu_envinit(struct samu_ctx *ctx) +{ + struct samu_environment *env; + + /* free old environments and pools in case we rebuilt the manifest */ + while (ctx->env.allenvs) { + env = ctx->env.allenvs; + ctx->env.allenvs = env->allnext; + } + + ctx->env.rootenv = samu_mkenv(ctx, NULL); + samu_envaddrule(ctx, ctx->env.rootenv, &ctx->phonyrule); + ctx->env.pools = NULL; + samu_addpool(ctx, &ctx->consolepool); +} + +static void +samu_addvar(struct samu_ctx *ctx, struct samu_treenode **tree, char *var, void *val) +{ + samu_treeinsert(ctx, tree, var, val); +} + +struct samu_environment * +samu_mkenv(struct samu_ctx *ctx, struct samu_environment *parent) +{ + struct samu_environment *env; + + env = samu_xmalloc(&ctx->arena, sizeof(*env)); + env->parent = parent; + env->bindings = NULL; + env->rules = NULL; + env->allnext = ctx->env.allenvs; + ctx->env.allenvs = env; + + return env; +} + +struct samu_string * +samu_envvar(struct samu_environment *env, char *var) +{ + struct samu_treenode *n; + + do { + n = samu_treefind(env->bindings, var); + if (n) + return n->value; + env = env->parent; + } while (env); + + return NULL; +} + +void +samu_envaddvar(struct samu_ctx *ctx, struct samu_environment *env, char *var, struct samu_string *val) +{ + samu_addvar(ctx, &env->bindings, var, val); +} + +static struct samu_string * +samu_merge(struct samu_ctx *ctx, struct samu_evalstring *str, size_t n) +{ + struct samu_string *result; + struct samu_evalstring *p; + char *s; + + result = samu_mkstr(&ctx->arena, n); + s = result->s; + for (p = str; p; p = p->next) { + if (!p->str) + continue; + memcpy(s, p->str->s, p->str->n); + s += p->str->n; + } + *s = '\0'; + + return result; +} + +struct samu_string * +samu_enveval(struct samu_ctx *ctx, struct samu_environment *env, struct samu_evalstring *str) +{ + size_t n; + struct samu_evalstring *p; + struct samu_string *res; + + n = 0; + for (p = str; p; p = p->next) { + if (p->var) + p->str = samu_envvar(env, p->var); + if (p->str) + n += p->str->n; + } + res = samu_merge(ctx, str, n); + samu_delevalstr(str); + + return res; +} + +void +samu_envaddrule(struct samu_ctx *ctx, struct samu_environment *env, struct samu_rule *r) +{ + if (samu_treeinsert(ctx, &env->rules, r->name, r)) + samu_fatal("rule '%s' redefined", r->name); +} + +struct samu_rule * +samu_envrule(struct samu_environment *env, char *name) +{ + struct samu_treenode *n; + + do { + n = samu_treefind(env->rules, name); + if (n) + return n->value; + env = env->parent; + } while (env); + + return NULL; +} + +static struct samu_string * +samu_pathlist(struct samu_ctx *ctx, struct samu_node **nodes, size_t n, char sep, bool escape) +{ + size_t i, len; + struct samu_string *path, *result; + char *s; + + if (n == 0) + return NULL; + if (n == 1) + return samu_nodepath(ctx, nodes[0], escape); + for (i = 0, len = 0; i < n; ++i) + len += samu_nodepath(ctx, nodes[i], escape)->n; + result = samu_mkstr(&ctx->arena, len + n - 1); + s = result->s; + for (i = 0; i < n; ++i) { + path = samu_nodepath(ctx, nodes[i], escape); + memcpy(s, path->s, path->n); + s += path->n; + *s++ = sep; + } + *--s = '\0'; + + return result; +} + +struct samu_rule * +samu_mkrule(struct samu_ctx *ctx, char *name) +{ + struct samu_rule *r; + + r = samu_xmalloc(&ctx->arena, sizeof(*r)); + r->name = name; + r->bindings = NULL; + + return r; +} + +void +samu_ruleaddvar(struct samu_ctx *ctx, struct samu_rule *r, char *var, struct samu_evalstring *val) +{ + samu_addvar(ctx, &r->bindings, var, val); +} + +struct samu_string * +samu_edgevar(struct samu_ctx *ctx, struct samu_edge *e, char *var, bool escape) +{ + static void *const cycle = (void *)&cycle; + struct samu_evalstring *str, *p; + struct samu_treenode *n; + size_t len; + + if (strcmp(var, "in") == 0) + return samu_pathlist(ctx, e->in, e->inimpidx, ' ', escape); + if (strcmp(var, "in_newline") == 0) + return samu_pathlist(ctx, e->in, e->inimpidx, '\n', escape); + if (strcmp(var, "out") == 0) + return samu_pathlist(ctx, e->out, e->outimpidx, ' ', escape); + n = samu_treefind(e->env->bindings, var); + if (n) + return n->value; + n = samu_treefind(e->rule->bindings, var); + if (!n) + return samu_envvar(e->env->parent, var); + if (n->value == cycle) + samu_fatal("cycle in rule variable involving '%s'", var); + str = n->value; + n->value = cycle; + len = 0; + for (p = str; p; p = p->next) { + if (p->var) + p->str = samu_edgevar(ctx, e, p->var, escape); + if (p->str) + len += p->str->n; + } + n->value = str; + return samu_merge(ctx, str, len); +} + +static void +samu_addpool(struct samu_ctx *ctx, struct samu_pool *p) +{ + if (samu_treeinsert(ctx, &ctx->env.pools, p->name, p)) + samu_fatal("pool '%s' redefined", p->name); +} + +struct samu_pool * +samu_mkpool(struct samu_ctx *ctx, char *name) +{ + struct samu_pool *p; + + p = samu_xmalloc(&ctx->arena, sizeof(*p)); + p->name = name; + p->numjobs = 0; + p->maxjobs = 0; + p->work = NULL; + samu_addpool(ctx, p); + + return p; +} + +struct samu_pool * +samu_poolget(struct samu_ctx *ctx, char *name) +{ + struct samu_treenode *n; + + n = samu_treefind(ctx->env.pools, name); + if (!n) + samu_fatal("unknown pool '%s'", name); + + return n->value; +} diff --git a/src/external/samurai/graph.c b/src/external/samurai/graph.c new file mode 100644 index 000000000..4e3618b47 --- /dev/null +++ b/src/external/samurai/graph.c @@ -0,0 +1,233 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/htab.h" +#include "external/samurai/util.h" + +static void +samu_delnode(void *p) +{ +} + +void +samu_graphinit(struct samu_ctx *ctx) +{ + struct samu_edge *e; + + /* delete old nodes and edges in case we rebuilt the manifest */ + samu_delhtab(ctx->graph.allnodes, samu_delnode); + while (ctx->graph.alledges) { + e = ctx->graph.alledges; + ctx->graph.alledges = e->allnext; + } + ctx->graph.allnodes = samu_mkhtab(&ctx->arena, 1024); +} + +struct samu_node * +samu_mknode(struct samu_ctx *ctx, struct samu_string *path) +{ + void **v; + struct samu_node *n; + struct samu_hashtablekey k; + + samu_htabkey(&k, path->s, path->n); + v = samu_htabput(&ctx->arena, ctx->graph.allnodes, &k); + if (*v) { + return *v; + } + n = samu_xmalloc(&ctx->arena, sizeof(*n)); + n->path = path; + n->shellpath = NULL; + n->gen = NULL; + n->use = NULL; + n->nuse = 0; + n->mtime = SAMU_MTIME_UNKNOWN; + n->logmtime = SAMU_MTIME_MISSING; + n->hash = 0; + n->id = -1; + *v = n; + + return n; +} + +struct samu_node * +samu_nodeget(struct samu_ctx *ctx, const char *path, size_t len) +{ + struct samu_hashtablekey k; + + if (!len) + len = strlen(path); + samu_htabkey(&k, path, len); + return samu_htabget(ctx->graph.allnodes, &k); +} + +void +samu_nodestat(struct samu_node *n) +{ + struct stat st; + + if (stat(n->path->s, &st) < 0) { + if (errno != ENOENT) + samu_fatal("stat %s:", n->path->s); + n->mtime = SAMU_MTIME_MISSING; + } else { +#ifdef __APPLE__ + n->mtime = (int64_t)st.st_mtime * 1000000000 + st.st_mtimensec; +/* +Illumos hides the members of st_mtim when you define _POSIX_C_SOURCE +since it has not been updated to support POSIX.1-2008: +https://www.illumos.org/issues/13327 +*/ +#elif defined(__sun) && !defined(__EXTENSIONS__) + n->mtime = (int64_t)st.st_mtim.__tv_sec * 1000000000 + st.st_mtim.__tv_nsec; +#else + n->mtime = (int64_t)st.st_mtim.tv_sec * 1000000000 + st.st_mtim.tv_nsec; +#endif + } +} + +struct samu_string * +samu_nodepath(struct samu_ctx *ctx, struct samu_node *n, bool escape) +{ + char *s, *d; + int nquote; + + if (!escape) + return n->path; + if (n->shellpath) + return n->shellpath; + escape = false; + nquote = 0; + for (s = n->path->s; *s; ++s) { + if (!isalnum(*(unsigned char *)s) && !strchr("_+-./", *s)) + escape = true; + if (*s == '\'') + ++nquote; + } + if (escape) { + n->shellpath = samu_mkstr(&ctx->arena, n->path->n + 2 + 3 * nquote); + d = n->shellpath->s; + *d++ = '\''; + for (s = n->path->s; *s; ++s) { + *d++ = *s; + if (*s == '\'') { + *d++ = '\\'; + *d++ = '\''; + *d++ = '\''; + } + } + *d++ = '\''; + } else { + n->shellpath = n->path; + } + return n->shellpath; +} + +void +samu_nodeuse(struct samu_ctx *ctx, struct samu_node *n, struct samu_edge *e) +{ + /* allocate in powers of two */ + if (!(n->nuse & (n->nuse - 1))) { + size_t new_nuse = n->nuse ? n->nuse * 2 : 1; + n->use = samu_xreallocarray(&ctx->arena, n->use, n->nuse, new_nuse, sizeof(e)); + } + n->use[n->nuse++] = e; +} + +struct samu_edge * +samu_mkedge(struct samu_ctx *ctx, struct samu_environment *parent) +{ + struct samu_edge *e; + + e = samu_xmalloc(&ctx->arena, sizeof(*e)); + e->env = samu_mkenv(ctx, parent); + e->pool = NULL; + e->out = NULL; + e->nout = 0; + e->in = NULL; + e->nin = 0; + e->flags = 0; + e->allnext = ctx->graph.alledges; + ctx->graph.alledges = e; + + return e; +} + +void +samu_edgehash(struct samu_ctx *ctx, struct samu_edge *e) +{ + static const char sep[] = ";rspfile="; + struct samu_string *cmd, *rsp, *s; + + if (e->flags & FLAG_HASH) + return; + e->flags |= FLAG_HASH; + cmd = samu_edgevar(ctx, e, "command", true); + if (!cmd) + samu_fatal("rule '%s' has no command", e->rule->name); + rsp = samu_edgevar(ctx, e, "rspfile_content", true); + if (rsp && rsp->n > 0) { + s = samu_mkstr(&ctx->arena, cmd->n + sizeof(sep) - 1 + rsp->n); + memcpy(s->s, cmd->s, cmd->n); + memcpy(s->s + cmd->n, sep, sizeof(sep) - 1); + memcpy(s->s + cmd->n + sizeof(sep) - 1, rsp->s, rsp->n); + s->s[s->n] = '\0'; + e->hash = samu_murmurhash64a(s->s, s->n); + } else { + e->hash = samu_murmurhash64a(cmd->s, cmd->n); + } +} + +static struct samu_edge * +samu_mkphony(struct samu_ctx *ctx, struct samu_node *n) +{ + struct samu_edge *e; + + e = samu_mkedge(ctx, ctx->env.rootenv); + e->rule = &ctx->phonyrule; + e->inimpidx = 0; + e->inorderidx = 0; + e->outimpidx = 1; + e->nout = 1; + e->out = samu_xmalloc(&ctx->arena, sizeof(n)); + e->out[0] = n; + + return e; +} + +void +samu_edgeadddeps(struct samu_ctx *ctx, struct samu_edge *e, struct samu_node **deps, size_t ndeps) +{ + struct samu_node **order, *n; + size_t norder, i; + + for (i = 0; i < ndeps; ++i) { + n = deps[i]; + if (!n->gen) + n->gen = samu_mkphony(ctx, n); + samu_nodeuse(ctx, n, e); + } + e->in = samu_xreallocarray(&ctx->arena, e->in, e->nin, e->nin + ndeps, sizeof(e->in[0])); + order = e->in + e->inorderidx; + norder = e->nin - e->inorderidx; + memmove(order + ndeps, order, norder * sizeof(e->in[0])); + memcpy(order, deps, ndeps * sizeof(e->in[0])); + e->inorderidx += ndeps; + e->nin += ndeps; +} diff --git a/src/external/samurai/htab.c b/src/external/samurai/htab.c new file mode 100644 index 000000000..dad28d2e7 --- /dev/null +++ b/src/external/samurai/htab.c @@ -0,0 +1,169 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/htab.h" +#include "external/samurai/util.h" + +struct samu_hashtable { + size_t len, cap; + struct samu_hashtablekey *keys; + void **vals; +}; + +void +samu_htabkey(struct samu_hashtablekey *k, const char *s, size_t n) +{ + k->str = s; + k->len = n; + k->hash = samu_murmurhash64a(s, n); +} + +struct samu_hashtable * +samu_mkhtab(struct samu_arena *a, size_t cap) +{ + struct samu_hashtable *h; + size_t i; + + assert(!(cap & (cap - 1))); + h = samu_xmalloc(a, sizeof(*h)); + h->len = 0; + h->cap = cap; + h->keys = samu_xreallocarray(a, NULL, 0, cap, sizeof(h->keys[0])); + h->vals = samu_xreallocarray(a, NULL, 0, cap, sizeof(h->vals[0])); + for (i = 0; i < cap; ++i) + h->keys[i].str = NULL; + + return h; +} + +void +samu_delhtab(struct samu_hashtable *h, void del(void *)) +{ + size_t i; + + if (!h) + return; + if (del) { + for (i = 0; i < h->cap; ++i) { + if (h->keys[i].str) + del(h->vals[i]); + } + } +} + +static bool +samu_keyequal(struct samu_hashtablekey *k1, struct samu_hashtablekey *k2) +{ + if (k1->hash != k2->hash || k1->len != k2->len) + return false; + return memcmp(k1->str, k2->str, k1->len) == 0; +} + +static size_t +samu_keyindex(struct samu_hashtable *h, struct samu_hashtablekey *k) +{ + size_t i; + + i = k->hash & (h->cap - 1); + while (h->keys[i].str && !samu_keyequal(&h->keys[i], k)) + i = (i + 1) & (h->cap - 1); + return i; +} + +void ** +samu_htabput(struct samu_arena *a, struct samu_hashtable *h, struct samu_hashtablekey *k) +{ + struct samu_hashtablekey *oldkeys; + void **oldvals; + size_t i, j, oldcap; + + if (h->cap / 2 < h->len) { + oldkeys = h->keys; + oldvals = h->vals; + oldcap = h->cap; + h->cap *= 2; + h->keys = samu_xreallocarray(a, NULL, 0, h->cap, sizeof(h->keys[0])); + h->vals = samu_xreallocarray(a, NULL, 0, h->cap, sizeof(h->vals[0])); + for (i = 0; i < h->cap; ++i) + h->keys[i].str = NULL; + for (i = 0; i < oldcap; ++i) { + if (oldkeys[i].str) { + j = samu_keyindex(h, &oldkeys[i]); + h->keys[j] = oldkeys[i]; + h->vals[j] = oldvals[i]; + } + } + } + i = samu_keyindex(h, k); + if (!h->keys[i].str) { + h->keys[i] = *k; + h->vals[i] = NULL; + ++h->len; + } + + return &h->vals[i]; +} + +void * +samu_htabget(struct samu_hashtable *h, struct samu_hashtablekey *k) +{ + size_t i; + + i = samu_keyindex(h, k); + return h->keys[i].str ? h->vals[i] : NULL; +} + +uint64_t +samu_murmurhash64a(const void *ptr, size_t len) +{ + const uint64_t seed = 0xdecafbaddecafbadull; + const uint64_t m = 0xc6a4a7935bd1e995ull; + uint64_t h, k, n; + const uint8_t *p, *end; + int r = 47; + + h = seed ^ (len * m); + n = len & ~0x7ull; + end = ptr; + end += n; + for (p = ptr; p != end; p += 8) { + memcpy(&k, p, sizeof(k)); + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + switch (len & 0x7) { + case 7: h ^= (uint64_t)p[6] << 48; /* fallthrough */ + case 6: h ^= (uint64_t)p[5] << 40; /* fallthrough */ + case 5: h ^= (uint64_t)p[4] << 32; /* fallthrough */ + case 4: h ^= (uint64_t)p[3] << 24; /* fallthrough */ + case 3: h ^= (uint64_t)p[2] << 16; /* fallthrough */ + case 2: h ^= (uint64_t)p[1] << 8; /* fallthrough */ + case 1: h ^= (uint64_t)p[0]; + h *= m; + } + + h ^= h >> r; + h *= m; + h ^= h >> r; + + return h; +} diff --git a/src/external/samurai/log.c b/src/external/samurai/log.c new file mode 100644 index 000000000..6d12d29aa --- /dev/null +++ b/src/external/samurai/log.c @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/graph.h" +#include "external/samurai/log.h" +#include "external/samurai/util.h" + +static const char *samu_logname = ".ninja_log"; +static const char *samu_logtmpname = ".ninja_log.tmp"; +static const char *samu_logfmt = "# ninja log v%d\n"; +static const int samu_logver = 5; + +static char * +samu_nextfield(char **end) +{ + char *s = *end; + + if (!*s) { + samu_warn("corrupt build log: missing field"); + return NULL; + } + *end += strcspn(*end, "\t\n"); + if (**end) + *(*end)++ = '\0'; + + return s; +} + +void +samu_loginit(struct samu_ctx *ctx, const char *builddir) +{ + int ver; + char *logpath = (char *)samu_logname, *logtmppath = (char *)samu_logtmpname, *p, *s; + size_t nline, nentry, i; + struct samu_edge *e; + struct samu_node *n; + int64_t mtime; + struct samu_buffer buf = {0}; + + nline = 0; + nentry = 0; + + if (ctx->log.logfile) { + fclose(ctx->log.logfile); + ctx->log.logfile = NULL; + } + if (builddir) + samu_xasprintf(&ctx->arena, &logpath, "%s/%s", builddir, samu_logname); + ctx->log.logfile = fopen(logpath, "r+"); + if (!ctx->log.logfile) { + if (errno != ENOENT) + samu_fatal("open %s:", logpath); + goto rewrite; + } + setvbuf(ctx->log.logfile, NULL, _IOLBF, 0); + if (fscanf(ctx->log.logfile, samu_logfmt, &ver) < 1) + goto rewrite; + if (ver != samu_logver) + goto rewrite; + + for (;;) { + if (buf.cap - buf.len < BUFSIZ) { + size_t newcap = buf.cap ? buf.cap * 2 : BUFSIZ; + buf.data = samu_xreallocarray(&ctx->arena, buf.data, buf.cap, newcap, 1); + buf.cap = newcap; + } + buf.data[buf.cap - 2] = '\0'; + if (!fgets(buf.data + buf.len, buf.cap - buf.len, ctx->log.logfile)) + break; + if (buf.data[buf.cap - 2] && buf.data[buf.cap - 2] != '\n') { + buf.len = buf.cap - 1; + continue; + } + ++nline; + p = buf.data; + buf.len = 0; + if (!samu_nextfield(&p)) /* start time */ + continue; + if (!samu_nextfield(&p)) /* end time */ + continue; + s = samu_nextfield(&p); /* mtime (used for restat) */ + if (!s) + continue; + mtime = strtoll(s, &s, 10); + if (*s) { + samu_warn("corrupt build log: invalid mtime"); + continue; + } + s = samu_nextfield(&p); /* output path */ + if (!s) + continue; + n = samu_nodeget(ctx, s, 0); + if (!n || !n->gen) + continue; + if (n->logmtime == SAMU_MTIME_MISSING) + ++nentry; + n->logmtime = mtime; + s = samu_nextfield(&p); /* command hash */ + if (!s) + continue; + n->hash = strtoull(s, &s, 16); + if (*s) { + samu_warn("corrupt build log: invalid hash for '%s'", n->path->s); + continue; + } + } + if (ferror(ctx->log.logfile)) { + samu_warn("build log read:"); + goto rewrite; + } + if (nline <= 100 || nline <= 3 * nentry) { + return; + } + +rewrite: + if (ctx->log.logfile) { + fclose(ctx->log.logfile); + ctx->log.logfile = NULL; + } + if (builddir) + samu_xasprintf(&ctx->arena, &logtmppath, "%s/%s", builddir, samu_logtmpname); + ctx->log.logfile = fopen(logtmppath, "w"); + if (!ctx->log.logfile) + samu_fatal("open %s:", logtmppath); + setvbuf(ctx->log.logfile, NULL, _IOLBF, 0); + fprintf(ctx->log.logfile, samu_logfmt, samu_logver); + if (nentry > 0) { + for (e = ctx->graph.alledges; e; e = e->allnext) { + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + if (!n->hash) + continue; + samu_logrecord(ctx, n); + } + } + } + fflush(ctx->log.logfile); + if (ferror(ctx->log.logfile)) + samu_fatal("build log write failed"); + if (rename(logtmppath, logpath) < 0) + samu_fatal("build log rename:"); +} + +void +samu_logclose(struct samu_ctx *ctx) +{ + fflush(ctx->log.logfile); + if (ferror(ctx->log.logfile)) + samu_fatal("build log write failed"); + fclose(ctx->log.logfile); + ctx->log.logfile = NULL; +} + +void +samu_logrecord(struct samu_ctx *ctx, struct samu_node *n) +{ + fprintf(ctx->log.logfile, "0\t0\t%" PRId64 "\t%s\t%" PRIx64 "\n", n->logmtime, n->path->s, n->hash); +} diff --git a/src/external/samurai/parse.c b/src/external/samurai/parse.c new file mode 100644 index 000000000..75cdce41f --- /dev/null +++ b/src/external/samurai/parse.c @@ -0,0 +1,282 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/parse.h" +#include "external/samurai/scan.h" +#include "external/samurai/util.h" + +void +samu_parseinit(struct samu_ctx *ctx) +{ + ctx->parse.deftarg = NULL; + ctx->parse.ndeftarg = 0; +} + +static void +samu_parselet(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_evalstring **val) +{ + samu_scanchar(s, '='); + *val = samu_scanstring(ctx, s, false); + samu_scannewline(s); +} + +static void +samu_parserule(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_environment *env) +{ + struct samu_rule *r; + char *var; + struct samu_evalstring *val; + bool hascommand = false, hasrspfile = false, hasrspcontent = false; + + r = samu_mkrule(ctx, samu_scanname(ctx, s)); + samu_scannewline(s); + while (samu_scanindent(s)) { + var = samu_scanname(ctx, s); + samu_parselet(ctx, s, &val); + samu_ruleaddvar(ctx, r, var, val); + if (!val) + continue; + if (strcmp(var, "command") == 0) + hascommand = true; + else if (strcmp(var, "rspfile") == 0) + hasrspfile = true; + else if (strcmp(var, "rspfile_content") == 0) + hasrspcontent = true; + } + if (!hascommand) + samu_fatal("rule '%s' has no command", r->name); + if (hasrspfile != hasrspcontent) + samu_fatal("rule '%s' has rspfile and no rspfile_content or vice versa", r->name); + samu_envaddrule(ctx, env, r); +} + +static void +samu_parseedge(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_environment *env) +{ + struct samu_edge *e; + struct samu_evalstring *str, **path; + char *name; + struct samu_string *val; + struct samu_node *n; + size_t i; + int p; + + e = samu_mkedge(ctx, env); + + samu_scanpaths(ctx, s); + e->outimpidx = ctx->scan.npaths; + if (samu_scanpipe(s, 1)) + samu_scanpaths(ctx, s); + e->nout = ctx->scan.npaths; + if (e->nout == 0) + samu_scanerror(s, "expected output path"); + samu_scanchar(s, ':'); + name = samu_scanname(ctx, s); + e->rule = samu_envrule(env, name); + if (!e->rule) + samu_fatal("undefined rule '%s'", name); + samu_scanpaths(ctx, s); + e->inimpidx = ctx->scan.npaths - e->nout; + p = samu_scanpipe(s, 1 | 2); + if (p == 1) { + samu_scanpaths(ctx, s); + p = samu_scanpipe(s, 2); + } + e->inorderidx = ctx->scan.npaths - e->nout; + if (p == 2) + samu_scanpaths(ctx, s); + e->nin = ctx->scan.npaths - e->nout; + samu_scannewline(s); + while (samu_scanindent(s)) { + name = samu_scanname(ctx, s); + samu_parselet(ctx, s, &str); + val = samu_enveval(ctx, env, str); + samu_envaddvar(ctx, e->env, name, val); + } + + e->out = samu_xreallocarray(&ctx->arena, NULL, 0, e->nout, sizeof(e->out[0])); + for (i = 0, path = ctx->scan.paths; i < e->nout; ++path) { + val = samu_enveval(ctx, e->env, *path); + samu_canonpath(val); + n = samu_mknode(ctx, val); + if (n->gen) { + if (!ctx->parseopts.dupbuildwarn) + samu_fatal("multiple rules generate '%s'", n->path->s); + samu_warn("multiple rules generate '%s'", n->path->s); + --e->nout; + if (i < e->outimpidx) + --e->outimpidx; + } else { + n->gen = e; + e->out[i] = n; + ++i; + } + } + e->in = samu_xreallocarray(&ctx->arena, NULL, 0, e->nin, sizeof(e->in[0])); + for (i = 0; i < e->nin; ++i, ++path) { + val = samu_enveval(ctx, e->env, *path); + samu_canonpath(val); + n = samu_mknode(ctx, val); + e->in[i] = n; + samu_nodeuse(ctx, n, e); + } + ctx->scan.npaths = 0; + + val = samu_edgevar(ctx, e, "pool", true); + if (val) + e->pool = samu_poolget(ctx, val->s); +} + +static void +samu_parseinclude(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_environment *env, bool newscope) +{ + struct samu_evalstring *str; + struct samu_string *path; + + str = samu_scanstring(ctx, s, true); + if (!str) + samu_scanerror(s, "expected include path"); + samu_scannewline(s); + path = samu_enveval(ctx, env, str); + + if (newscope) + env = samu_mkenv(ctx, env); + samu_parse(ctx, path->s, env); +} + +static void +samu_parsedefault(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_environment *env) +{ + struct samu_string *path; + struct samu_node *n; + size_t i; + + samu_scanpaths(ctx, s); + ctx->parse.deftarg = samu_xreallocarray(&ctx->arena, ctx->parse.deftarg, ctx->parse.ndeftarg, ctx->parse.ndeftarg + ctx->scan.npaths, sizeof(*ctx->parse.deftarg)); + for (i = 0; i < ctx->scan.npaths; ++i) { + path = samu_enveval(ctx, env, ctx->scan.paths[i]); + samu_canonpath(path); + n = samu_nodeget(ctx, path->s, path->n); + if (!n) + samu_fatal("unknown target '%s'", path->s); + ctx->parse.deftarg[ctx->parse.ndeftarg++] = n; + } + samu_scannewline(s); + ctx->scan.npaths = 0; +} + +static void +samu_parsepool(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_environment *env) +{ + struct samu_pool *p; + struct samu_evalstring *val; + struct samu_string *str; + char *var, *end; + + p = samu_mkpool(ctx, samu_scanname(ctx, s)); + samu_scannewline(s); + while (samu_scanindent(s)) { + var = samu_scanname(ctx, s); + samu_parselet(ctx, s, &val); + if (strcmp(var, "depth") == 0) { + str = samu_enveval(ctx, env, val); + p->maxjobs = strtol(str->s, &end, 10); + if (*end) + samu_fatal("invalid pool depth '%s'", str->s); + } else { + samu_fatal("unexpected pool variable '%s'", var); + } + } + if (!p->maxjobs) + samu_fatal("pool '%s' has no depth", p->name); +} + +static void +samu_checkversion(const char *ver) +{ + int major, minor = 0; + + if (sscanf(ver, "%d.%d", &major, &minor) < 1) + samu_fatal("invalid ninja_required_version"); + if (major > ninjamajor || (major == ninjamajor && minor > ninjaminor)) + samu_fatal("ninja_required_version %s is newer than %d.%d", ver, ninjamajor, ninjaminor); +} + +void +samu_parse(struct samu_ctx *ctx, const char *name, struct samu_environment *env) +{ + struct samu_scanner s; + char *var; + struct samu_string *val; + struct samu_evalstring *str; + + samu_scaninit(&s, name); + for (;;) { + switch (samu_scankeyword(ctx, &s, &var)) { + case SAMU_RULE: + samu_parserule(ctx, &s, env); + break; + case SAMU_BUILD: + samu_parseedge(ctx, &s, env); + break; + case SAMU_INCLUDE: + samu_parseinclude(ctx, &s, env, false); + break; + case SAMU_SUBNINJA: + samu_parseinclude(ctx, &s, env, true); + break; + case SAMU_DEFAULT: + samu_parsedefault(ctx, &s, env); + break; + case SAMU_POOL: + samu_parsepool(ctx, &s, env); + break; + case SAMU_VARIABLE: + samu_parselet(ctx, &s, &str); + val = samu_enveval(ctx, env, str); + if (strcmp(var, "ninja_required_version") == 0) + samu_checkversion(val->s); + samu_envaddvar(ctx, env, var, val); + break; + case EOF: + samu_scanclose(&s); + return; + } + } +} + +void +samu_defaultnodes(struct samu_ctx *ctx, void fn(struct samu_ctx *ctx, struct samu_node *)) +{ + struct samu_edge *e; + struct samu_node *n; + size_t i; + + if (ctx->parse.ndeftarg > 0) { + for (i = 0; i < ctx->parse.ndeftarg; ++i) + fn(ctx, ctx->parse.deftarg[i]); + } else { + /* by default build all nodes which are not used by any edges */ + for (e = ctx->graph.alledges; e; e = e->allnext) { + for (i = 0; i < e->nout; ++i) { + n = e->out[i]; + if (n->nuse == 0) + fn(ctx, n); + } + } + } +} diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c new file mode 100644 index 000000000..5026b2e78 --- /dev/null +++ b/src/external/samurai/samu.c @@ -0,0 +1,300 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include +#include /* for chdir */ + +#include "buf_size.h" +#include "external/samurai/ctx.h" + +#include "external/samurai/arg.h" +#include "external/samurai/build.h" +#include "external/samurai/deps.h" +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/log.h" +#include "external/samurai/parse.h" +#include "external/samurai/samu.h" +#include "external/samurai/tool.h" +#include "external/samurai/util.h" + +static void +samu_usage(struct samu_ctx *ctx) +{ + fprintf(stderr, "usage: %s [-C dir] [-f buildfile] [-j maxjobs] [-k maxfail] [-l maxload] [-n]\n", ctx->argv0); + exit(2); +} + +static char * +samu_getbuilddir(struct samu_ctx *ctx) +{ + struct samu_string *builddir; + + builddir = samu_envvar(ctx->env.rootenv, "builddir"); + if (!builddir) + return NULL; + if (samu_makedirs(builddir, false) < 0) + exit(1); + return builddir->s; +} + +static void +samu_debugflag(struct samu_ctx *ctx, const char *flag) +{ + if (strcmp(flag, "explain") == 0) + ctx->buildopts.explain = true; + else if (strcmp(flag, "keepdepfile") == 0) + ctx->buildopts.keepdepfile = true; + else if (strcmp(flag, "keeprsp") == 0) + ctx->buildopts.keeprsp = true; + else + samu_fatal("unknown debug flag '%s'", flag); +} + +static void +samu_loadflag(struct samu_ctx *ctx, const char *flag) +{ +#ifdef NO_GETLOADAVG + samu_warn("job scheduling based on load average is not implemented"); +#else + double value; + char *end; + errno = 0; + + value = strtod(flag, &end); + if (*end || value < 0 || errno != 0) + samu_fatal("invalid -l parameter"); + ctx->buildopts.maxload = value; +#endif +} + +static void +samu_warnflag(struct samu_ctx *ctx, const char *flag) +{ + if (strcmp(flag, "dupbuild=err") == 0) + ctx->parseopts.dupbuildwarn = false; + else if (strcmp(flag, "dupbuild=warn") == 0) + ctx->parseopts.dupbuildwarn = true; + else + samu_fatal("unknown warning flag '%s'", flag); +} + +static void +samu_jobsflag(struct samu_ctx *ctx, const char *flag) +{ + long num; + char *end; + + num = strtol(flag, &end, 10); + if (*end || num < 0) + samu_fatal("invalid -j parameter"); + ctx->buildopts.maxjobs = num > 0 ? num : -1; +} + +static void +samu_parseenvargs(struct samu_ctx *ctx, char *env) +{ + char *arg, *argvbuf[64], **argv = argvbuf; + int argc; + + if (!env) + return; + env = samu_xmemdup(&ctx->arena, env, strlen(env) + 1); + argc = 1; + argv[0] = NULL; + arg = strtok(env, " "); + while (arg) { + if ((size_t)argc >= ARRAY_LEN(argvbuf) - 1) + samu_fatal("too many arguments in SAMUFLAGS"); + argv[argc++] = arg; + arg = strtok(NULL, " "); + } + argv[argc] = NULL; + + SAMU_ARGBEGIN { + case 'j': + samu_jobsflag(ctx, SAMU_EARGF(samu_usage(ctx))); + break; + case 'v': + ctx->buildopts.verbose = true; + break; + case 'l': + samu_loadflag(ctx, SAMU_EARGF(samu_usage(ctx))); + break; + default: + samu_fatal("invalid option in SAMUFLAGS"); + } SAMU_ARGEND +} + +static const char * +samu_progname(const char *arg, const char *def) +{ + const char *slash; + + if (!arg) + return def; + slash = strrchr(arg, '/'); + return slash ? slash + 1 : arg; +} + +int +samu_main(int argc, char *argv[]) +{ + char *builddir, *manifest = "build.ninja", *end, *arg; + const struct samu_tool *tool = NULL; + struct samu_node *n; + long num; + int tries; + + struct samu_ctx samu_ctx = { + .buildopts = {.maxfail = 1}, + .phonyrule = {.name = "phony"}, + .consolepool = {.name = "console", .maxjobs = 1}, + }, *ctx = &samu_ctx; + + samu_arena_init(&ctx->arena); + + ctx->argv0 = samu_progname(argv[0], "samu"); + samu_parseenvargs(ctx, getenv("SAMUFLAGS")); + SAMU_ARGBEGIN { + case '-': + arg = SAMU_EARGF(samu_usage(ctx)); + if (strcmp(arg, "version") == 0) { + printf("%d.%d.0\n", ninjamajor, ninjaminor); + return 0; + } else if (strcmp(arg, "verbose") == 0) { + ctx->buildopts.verbose = true; + } else { + samu_usage(ctx); + } + break; + case 'C': + arg = SAMU_EARGF(samu_usage(ctx)); + /* samu_warn("entering directory '%s'", arg); */ + if (chdir(arg) < 0) + samu_fatal("chdir:"); + break; + case 'd': + samu_debugflag(ctx, SAMU_EARGF(samu_usage(ctx))); + break; + case 'f': + manifest = SAMU_EARGF(samu_usage(ctx)); + break; + case 'j': + samu_jobsflag(ctx, SAMU_EARGF(samu_usage(ctx))); + break; + case 'k': + num = strtol(SAMU_EARGF(samu_usage(ctx)), &end, 10); + if (*end) + samu_fatal("invalid -k parameter"); + ctx->buildopts.maxfail = num > 0 ? num : -1; + break; + case 'l': + samu_loadflag(ctx, SAMU_EARGF(samu_usage(ctx))); + break; + case 'n': + ctx->buildopts.dryrun = true; + break; + case 't': + tool = samu_toolget(SAMU_EARGF(samu_usage(ctx))); + goto argdone; + case 'v': + ctx->buildopts.verbose = true; + break; + case 'w': + samu_warnflag(ctx, SAMU_EARGF(samu_usage(ctx))); + break; + default: + samu_usage(ctx); + } SAMU_ARGEND +argdone: + if (!ctx->buildopts.maxjobs) { +#ifdef _SC_NPROCESSORS_ONLN + int n = sysconf(_SC_NPROCESSORS_ONLN); + switch (n) { + case -1: case 0: case 1: + ctx->buildopts.maxjobs = 2; + break; + case 2: + ctx->buildopts.maxjobs = 3; + break; + default: + ctx->buildopts.maxjobs = n + 2; + break; + } +#else + ctx->buildopts.maxjobs = 2; +#endif + } + + ctx->buildopts.statusfmt = getenv("NINJA_STATUS"); + if (!ctx->buildopts.statusfmt) + ctx->buildopts.statusfmt = "[%s/%t] "; + + setvbuf(stdout, NULL, _IOLBF, 0); + + tries = 0; +retry: + /* (re-)initialize global graph, environment, and parse structures */ + samu_graphinit(ctx); + samu_envinit(ctx); + samu_parseinit(ctx); + + /* parse the manifest */ + samu_parse(ctx, manifest, ctx->env.rootenv); + + if (tool) { + int r = tool->run(ctx, argc, argv); + samu_arena_destroy(&ctx->arena); + return r; + } + + /* load the build log */ + builddir = samu_getbuilddir(ctx); + samu_loginit(ctx, builddir); + samu_depsinit(ctx, builddir); + + /* rebuild the manifest if it's dirty */ + n = samu_nodeget(ctx, manifest, 0); + if (n && n->gen) { + samu_buildadd(ctx, n); + if (n->dirty) { + samu_build(ctx); + if (n->gen->flags & FLAG_DIRTY_OUT || n->gen->nprune > 0) { + if (++tries > 100) + samu_fatal("manifest '%s' dirty after 100 tries", manifest); + if (!ctx->buildopts.dryrun) + goto retry; + } + /* manifest was pruned; reset state, then continue with build */ + samu_buildreset(ctx); + } + } + + /* finally, build any specified targets or the default targets */ + if (argc) { + for (; *argv; ++argv) { + n = samu_nodeget(ctx, *argv, 0); + if (!n) + samu_fatal("unknown target '%s'", *argv); + samu_buildadd(ctx, n); + } + } else { + samu_defaultnodes(ctx, samu_buildadd); + } + samu_build(ctx); + samu_logclose(ctx); + samu_depsclose(ctx); + + samu_arena_destroy(&ctx->arena); + return 0; +} diff --git a/src/external/samurai/scan.c b/src/external/samurai/scan.c new file mode 100644 index 000000000..28e94388e --- /dev/null +++ b/src/external/samurai/scan.c @@ -0,0 +1,362 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include +#include + +#include "buf_size.h" +#include "external/samurai/ctx.h" + +#include "external/samurai/scan.h" +#include "external/samurai/util.h" + +#undef getc +#define getc getc_unlocked + +void +samu_scaninit(struct samu_scanner *s, const char *path) +{ + s->path = path; + s->line = 1; + s->col = 1; + s->f = fopen(path, "r"); + if (!s->f) + samu_fatal("open %s:", path); + s->chr = getc(s->f); +} + +void +samu_scanclose(struct samu_scanner *s) +{ + fclose(s->f); +} + +void +samu_scanerror(struct samu_scanner *s, const char *fmt, ...) +{ + va_list ap; + + fprintf(stderr, "samu: %s:%d:%d: ", s->path, s->line, s->col); + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + putc('\n', stderr); + exit(1); +} + +static int +samu_next(struct samu_scanner *s) +{ + if (s->chr == '\n') { + ++s->line; + s->col = 1; + } else { + ++s->col; + } + s->chr = getc(s->f); + + return s->chr; +} + +static int +samu_issimplevar(int c) +{ + return isalnum(c) || c == '_' || c == '-'; +} + +static int +samu_isvar(int c) +{ + return samu_issimplevar(c) || c == '.'; +} + +static bool +samu_newline(struct samu_scanner *s) +{ + switch (s->chr) { + case '\r': + if (samu_next(s) != '\n') + samu_scanerror(s, "expected '\\n' after '\\r'"); + /* fallthrough */ + case '\n': + samu_next(s); + return true; + } + return false; +} + +static bool +samu_singlespace(struct samu_scanner *s) +{ + switch (s->chr) { + case '$': + samu_next(s); + if (samu_newline(s)) + return true; + ungetc(s->chr, s->f); + s->chr = '$'; + return false; + case ' ': + samu_next(s); + return true; + } + return false; +} + +static bool +samu_space(struct samu_scanner *s) +{ + if (!samu_singlespace(s)) + return false; + while (samu_singlespace(s)) + ; + return true; +} + +static bool +samu_comment(struct samu_scanner *s) +{ + if (s->chr != '#') + return false; + do samu_next(s); + while (!samu_newline(s)); + return true; +} + +static void +samu_name(struct samu_ctx *ctx, struct samu_scanner *s) +{ + ctx->scan.buf.len = 0; + while (samu_isvar(s->chr)) { + samu_bufadd(&ctx->arena, &ctx->scan.buf, s->chr); + samu_next(s); + } + if (!ctx->scan.buf.len) + samu_scanerror(s, "expected name"); + samu_bufadd(&ctx->arena, &ctx->scan.buf, '\0'); + samu_space(s); +} + +int +samu_scankeyword(struct samu_ctx *ctx, struct samu_scanner *s, char **var) +{ + /* must stay in sorted order */ + static const struct { + const char *name; + int value; + } keywords[] = { + {"build", SAMU_BUILD}, + {"default", SAMU_DEFAULT}, + {"include", SAMU_INCLUDE}, + {"pool", SAMU_POOL}, + {"rule", SAMU_RULE}, + {"subninja", SAMU_SUBNINJA}, + }; + int low = 0, high = ARRAY_LEN(keywords) - 1, mid, cmp; + + for (;;) { + switch (s->chr) { + case ' ': + samu_space(s); + if (!samu_comment(s) && !samu_newline(s)) + samu_scanerror(s, "unexpected indent"); + break; + case '#': + samu_comment(s); + break; + case '\r': + case '\n': + samu_newline(s); + break; + case EOF: + return EOF; + default: + samu_name(ctx, s); + while (low <= high) { + mid = (low + high) / 2; + cmp = strcmp(ctx->scan.buf.data, keywords[mid].name); + if (cmp == 0) + return keywords[mid].value; + if (cmp < 0) + high = mid - 1; + else + low = mid + 1; + } + *var = samu_xmemdup(&ctx->arena, ctx->scan.buf.data, ctx->scan.buf.len); + return SAMU_VARIABLE; + } + } +} + +char * +samu_scanname(struct samu_ctx *ctx, struct samu_scanner *s) +{ + samu_name(ctx, s); + return samu_xmemdup(&ctx->arena, ctx->scan.buf.data, ctx->scan.buf.len); +} + +static void +samu_addstringpart(struct samu_ctx *ctx, struct samu_evalstring ***end, bool var) +{ + struct samu_evalstring *p; + + p = samu_xmalloc(&ctx->arena, sizeof(*p)); + p->next = NULL; + **end = p; + if (var) { + samu_bufadd(&ctx->arena, &ctx->scan.buf, '\0'); + p->var = samu_xmemdup(&ctx->arena, ctx->scan.buf.data, ctx->scan.buf.len); + } else { + p->var = NULL; + p->str = samu_mkstr(&ctx->arena, ctx->scan.buf.len); + memcpy(p->str->s, ctx->scan.buf.data, ctx->scan.buf.len); + p->str->s[ctx->scan.buf.len] = '\0'; + } + *end = &p->next; + ctx->scan.buf.len = 0; +} + +static void +samu_escape(struct samu_ctx *ctx, struct samu_scanner *s, struct samu_evalstring ***end) +{ + switch (s->chr) { + case '$': + case ' ': + case ':': + samu_bufadd(&ctx->arena, &ctx->scan.buf, s->chr); + samu_next(s); + break; + case '{': + if (ctx->scan.buf.len > 0) + samu_addstringpart(ctx, end, false); + while (samu_isvar(samu_next(s))) + samu_bufadd(&ctx->arena, &ctx->scan.buf, s->chr); + if (s->chr != '}') + samu_scanerror(s, "invalid variable name"); + samu_next(s); + samu_addstringpart(ctx, end, true); + break; + case '\r': + case '\n': + samu_newline(s); + samu_space(s); + break; + default: + if (ctx->scan.buf.len > 0) + samu_addstringpart(ctx, end, false); + while (samu_issimplevar(s->chr)) { + samu_bufadd(&ctx->arena, &ctx->scan.buf, s->chr); + samu_next(s); + } + if (!ctx->scan.buf.len) + samu_scanerror(s, "invalid $ escape"); + samu_addstringpart(ctx, end, true); + } +} + +struct samu_evalstring * +samu_scanstring(struct samu_ctx *ctx, struct samu_scanner *s, bool path) +{ + struct samu_evalstring *str = NULL, **end = &str; + + ctx->scan.buf.len = 0; + for (;;) { + switch (s->chr) { + case '$': + samu_next(s); + samu_escape(ctx, s, &end); + break; + case ':': + case '|': + case ' ': + if (path) + goto out; + /* fallthrough */ + default: + samu_bufadd(&ctx->arena, &ctx->scan.buf, s->chr); + samu_next(s); + break; + case '\r': + case '\n': + case EOF: + goto out; + } + } +out: + if (ctx->scan.buf.len > 0) + samu_addstringpart(ctx, &end, 0); + if (path) + samu_space(s); + return str; +} + +void +samu_scanpaths(struct samu_ctx *ctx, struct samu_scanner *s) +{ + struct samu_evalstring *str; + + while ((str = samu_scanstring(ctx, s, true))) { + if (ctx->scan.npaths == ctx->scan.paths_max) { + size_t newmax = ctx->scan.paths_max ? ctx->scan.paths_max * 2 : 32; + ctx->scan.paths = samu_xreallocarray(&ctx->arena, ctx->scan.paths, ctx->scan.paths_max, newmax, sizeof(ctx->scan.paths[0])); + ctx->scan.paths_max = newmax; + } + ctx->scan.paths[ctx->scan.npaths++] = str; + } +} + +void +samu_scanchar(struct samu_scanner *s, int c) +{ + if (s->chr != c) + samu_scanerror(s, "expected '%c'", c); + samu_next(s); + samu_space(s); +} + +int +samu_scanpipe(struct samu_scanner *s, int n) +{ + if (s->chr != '|') + return 0; + samu_next(s); + if (s->chr != '|') { + if (!(n & 1)) + samu_scanerror(s, "expected '||'"); + samu_space(s); + return 1; + } + if (!(n & 2)) + samu_scanerror(s, "unexpected '||'"); + samu_next(s); + samu_space(s); + return 2; +} + +bool +samu_scanindent(struct samu_scanner *s) +{ + bool indent; + + for (;;) { + indent = samu_space(s); + if (!samu_comment(s)) + return indent && !samu_newline(s); + } +} + +void +samu_scannewline(struct samu_scanner *s) +{ + if (!samu_newline(s)) + samu_scanerror(s, "expected newline"); +} diff --git a/src/external/samurai/tool.c b/src/external/samurai/tool.c new file mode 100644 index 000000000..29da424fb --- /dev/null +++ b/src/external/samurai/tool.c @@ -0,0 +1,475 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "buf_size.h" +#include "external/samurai/ctx.h" + +#include "external/samurai/arg.h" +#include "external/samurai/env.h" +#include "external/samurai/graph.h" +#include "external/samurai/parse.h" +#include "external/samurai/tool.h" +#include "external/samurai/util.h" + +static int +samu_cleanpath(struct samu_string *path) +{ + if (path) { + if (remove(path->s) == 0) { + printf("remove %s\n", path->s); + } else if (errno != ENOENT) { + samu_warn("remove %s:", path->s); + return -1; + } + } + + return 0; +} + +static int +samu_cleanedge(struct samu_ctx *ctx, struct samu_edge *e) +{ + int ret = 0; + size_t i; + + for (i = 0; i < e->nout; ++i) { + if (samu_cleanpath(e->out[i]->path) < 0) + ret = -1; + } + if (samu_cleanpath(samu_edgevar(ctx, e, "rspfile", false)) < 0) + ret = -1; + if (samu_cleanpath(samu_edgevar(ctx, e, "depfile", false)) < 0) + ret = -1; + + return ret; +} + +static int +samu_cleantarget(struct samu_ctx *ctx, struct samu_node *n) +{ + int ret = 0; + size_t i; + + if (!n->gen || n->gen->rule == &ctx->phonyrule) + return 0; + if (samu_cleanpath(n->path) < 0) + ret = -1; + for (i = 0; i < n->gen->nin; ++i) { + if (samu_cleantarget(ctx, n->gen->in[i]) < 0) + ret = -1; + } + + return ret; +} + +static int +samu_clean(struct samu_ctx *ctx, int argc, char *argv[]) +{ + int ret = 0; + bool cleangen = false, cleanrule = false; + struct samu_edge *e; + struct samu_node *n; + struct samu_rule *r; + + SAMU_ARGBEGIN { + case 'g': + cleangen = true; + break; + case 'r': + cleanrule = true; + break; + default: + fprintf(stderr, "usage: %s ... -t clean [-gr] [targets...]\n", ctx->argv0); + return 2; + } SAMU_ARGEND + + if (cleanrule) { + if (!argc) + samu_fatal("expected a rule to clean"); + for (; *argv; ++argv) { + r = samu_envrule(ctx->env.rootenv, *argv); + if (!r) { + samu_warn("unknown rule '%s'", *argv); + ret = 1; + continue; + } + for (e = ctx->graph.alledges; e; e = e->allnext) { + if (e->rule != r) + continue; + if (samu_cleanedge(ctx, e) < 0) + ret = 1; + } + } + } else if (argc > 0) { + for (; *argv; ++argv) { + n = samu_nodeget(ctx, *argv, 0); + if (!n) { + samu_warn("unknown target '%s'", *argv); + ret = 1; + continue; + } + if (samu_cleantarget(ctx, n) < 0) + ret = 1; + } + } else { + for (e = ctx->graph.alledges; e; e = e->allnext) { + if (e->rule == &ctx->phonyrule) + continue; + if (!cleangen && samu_edgevar(ctx, e, "generator", true)) + continue; + if (samu_cleanedge(ctx, e) < 0) + ret = 1; + } + } + + return ret; +} + +/* depth-first traversal */ +static void +samu_targetcommands(struct samu_ctx *ctx, struct samu_node *n) +{ + struct samu_edge *e = n->gen; + struct samu_string *command; + size_t i; + + if (!e || (e->flags & FLAG_WORK)) + return; + e->flags |= FLAG_WORK; + for (i = 0; i < e->nin; ++i) + samu_targetcommands(ctx, e->in[i]); + command = samu_edgevar(ctx, e, "command", true); + if (command && command->n) + puts(command->s); +} + +static int +samu_commands(struct samu_ctx *ctx, int argc, char *argv[]) +{ + struct samu_node *n; + + if (argc > 1) { + while (*++argv) { + n = samu_nodeget(ctx, *argv, 0); + if (!n) + samu_fatal("unknown target '%s'", *argv); + samu_targetcommands(ctx, n); + } + } else { + samu_defaultnodes(ctx, samu_targetcommands); + } + + if (fflush(stdout) || ferror(stdout)) + samu_fatal("write failed"); + + return 0; +} + +static void +samu_printjson(const char *s, size_t n, bool join) +{ + size_t i; + char c; + + for (i = 0; i < n; ++i) { + c = s[i]; + switch (c) { + case '"': + case '\\': + putchar('\\'); + break; + case '\n': + if (join) + c = ' '; + break; + case '\0': + return; + } + putchar(c); + } +} + +static int +samu_compdb(struct samu_ctx *ctx, int argc, char *argv[]) +{ + char dir[PATH_MAX], *p; + struct samu_edge *e; + struct samu_string *cmd, *rspfile, *content; + bool expandrsp = false, first = true; + int i; + size_t off; + + SAMU_ARGBEGIN { + case 'x': + expandrsp = true; + break; + default: + fprintf(stderr, "usage: %s ... -t compdb [-x] [rules...]\n", ctx->argv0); + return 2; + } SAMU_ARGEND + + if (!getcwd(dir, sizeof(dir))) + samu_fatal("getcwd:"); + + putchar('['); + for (e = ctx->graph.alledges; e; e = e->allnext) { + if (e->nin == 0) + continue; + for (i = 0; i < argc; ++i) { + if (strcmp(e->rule->name, argv[i]) == 0) { + if (first) + first = false; + else + putchar(','); + + printf("\n {\n \"directory\": \""); + samu_printjson(dir, -1, false); + + printf("\",\n \"command\": \""); + cmd = samu_edgevar(ctx, e, "command", true); + rspfile = expandrsp ? samu_edgevar(ctx, e, "rspfile", true) : NULL; + p = rspfile ? strstr(cmd->s, rspfile->s) : NULL; + if (!p || p == cmd->s || p[-1] != '@') { + samu_printjson(cmd->s, cmd->n, false); + } else { + off = p - cmd->s; + samu_printjson(cmd->s, off - 1, false); + content = samu_edgevar(ctx, e, "rspfile_content", true); + samu_printjson(content->s, content->n, true); + off += rspfile->n; + samu_printjson(cmd->s + off, cmd->n - off, false); + } + + printf("\",\n \"file\": \""); + samu_printjson(e->in[0]->path->s, -1, false); + + printf("\",\n \"output\": \""); + samu_printjson(e->out[0]->path->s, -1, false); + + printf("\"\n }"); + break; + } + } + } + puts("\n]"); + + if (fflush(stdout) || ferror(stdout)) + samu_fatal("write failed"); + + return 0; +} + +static void +samu_graphnode(struct samu_ctx *ctx, struct samu_node *n) +{ + struct samu_edge *e = n->gen; + size_t i; + const char *style; + + printf("\"%p\" [label=\"%s\"]\n", (void *)n, n->path->s); + + if (!e || (e->flags & FLAG_WORK)) + return; + e->flags |= FLAG_WORK; + + for (i = 0; i < e->nin; ++i) + samu_graphnode(ctx, e->in[i]); + + if (e->nin == 1 && e->nout == 1) { + printf("\"%p\" -> \"%p\" [label=\"%s\"]\n", (void *)e->in[0], (void *)e->out[0], e->rule->name); + } else { + printf("\"%p\" [label=\"%s\", shape=ellipse]\n", (void *)e, e->rule->name); + for (i = 0; i < e->nout; ++i) + printf("\"%p\" -> \"%p\"\n", (void *)e, (void *)e->out[i]); + for (i = 0; i < e->nin; ++i) { + style = i >= e->inorderidx ? " style=dotted" : ""; + printf("\"%p\" -> \"%p\" [arrowhead=none%s]\n", (void *)e->in[i], (void *)e, style); + } + } +} + +static int +samu_graph(struct samu_ctx *ctx, int argc, char *argv[]) +{ + struct samu_node *n; + + puts("digraph ninja {"); + puts("rankdir=\"LR\""); + puts("node [fontsize=10, shape=box, height=0.25]"); + puts("edge [fontsize=10]"); + + if (argc > 1) { + while (*++argv) { + n = samu_nodeget(ctx, *argv, 0); + if (!n) + samu_fatal("unknown target '%s'", *argv); + samu_graphnode(ctx, n); + } + } else { + samu_defaultnodes(ctx, samu_graphnode); + } + + puts("}"); + + if (fflush(stdout) || ferror(stdout)) + samu_fatal("write failed"); + + return 0; +} + +static int +samu_query(struct samu_ctx *ctx, int argc, char *argv[]) +{ + struct samu_node *n; + struct samu_edge *e; + char *path; + int i; + size_t j, k; + + if (argc == 1) { + fprintf(stderr, "usage: %s ... -t query target...\n", ctx->argv0); + exit(2); + } + for (i = 1; i < argc; ++i) { + path = argv[i]; + n = samu_nodeget(ctx, path, 0); + if (!n) + samu_fatal("unknown target '%s'", path); + printf("%s:\n", argv[i]); + e = n->gen; + if (e) { + printf(" input: %s\n", e->rule->name); + for (j = 0; j < e->nin; ++j) + printf(" %s\n", e->in[j]->path->s); + } + puts(" outputs:"); + for (j = 0; j < n->nuse; ++j) { + e = n->use[j]; + for (k = 0; k < e->nout; ++k) + printf(" %s\n", e->out[k]->path->s); + } + } + + return 0; +} + +static void +samu_targetsdepth(struct samu_node *n, size_t depth, size_t indent) +{ + struct samu_edge *e = n->gen; + size_t i; + + for (i = 0; i < indent; ++i) + printf(" "); + if (e) { + printf("%s: %s\n", n->path->s, e->rule->name); + if (depth != 1) { + for (i = 0; i < e->nin; ++i) + samu_targetsdepth(e->in[i], depth - 1, indent + 1); + } + } else { + puts(n->path->s); + } +} + +static void +samu_targetsusage(struct samu_ctx *ctx) +{ + fprintf(stderr, + "usage: %s ... -t targets [depth [maxdepth]]\n" + " %s ... -t targets rule [rulename]\n" + " %s ... -t targets all\n", + ctx->argv0, ctx->argv0, ctx->argv0); + exit(2); +} + +static int +samu_targets(struct samu_ctx *ctx, int argc, char *argv[]) +{ + struct samu_edge *e; + size_t depth = 1, i; + char *end, *mode, *name; + + if (argc > 3) + samu_targetsusage(ctx); + mode = argv[1]; + if (!mode || strcmp(mode, "depth") == 0) { + if (argc == 3) { + depth = strtol(argv[2], &end, 10); + if (*end) + samu_targetsusage(ctx); + } + for (e = ctx->graph.alledges; e; e = e->allnext) { + for (i = 0; i < e->nout; ++i) { + if (e->out[i]->nuse == 0) + samu_targetsdepth(e->out[i], depth, 0); + } + } + } else if (strcmp(mode, "rule") == 0) { + name = argv[2]; + for (e = ctx->graph.alledges; e; e = e->allnext) { + if (!name) { + for (i = 0; i < e->nin; ++i) { + if (!e->in[i]->gen) + puts(e->in[i]->path->s); + } + } else if (strcmp(e->rule->name, name) == 0) { + for (i = 0; i < e->nout; ++i) + puts(e->out[i]->path->s); + } + } + } else if (strcmp(mode, "all") == 0 && argc == 2) { + for (e = ctx->graph.alledges; e; e = e->allnext) { + for (i = 0; i < e->nout; ++i) + printf("%s: %s\n", e->out[i]->path->s, e->rule->name); + } + } else { + samu_targetsusage(ctx); + } + + if (fflush(stdout) || ferror(stdout)) + samu_fatal("write failed"); + + return 0; +} + +const struct samu_tool * +samu_toolget(const char *name) +{ + static const struct samu_tool tools[] = { + {"clean", samu_clean}, + {"commands", samu_commands}, + {"compdb", samu_compdb}, + {"graph", samu_graph}, + {"query", samu_query}, + {"targets", samu_targets}, + }; + + const struct samu_tool *t; + size_t i; + + t = NULL; + for (i = 0; i < ARRAY_LEN(tools); ++i) { + if (strcmp(name, tools[i].name) == 0) { + t = &tools[i]; + break; + } + } + if (!t) + samu_fatal("unknown tool '%s'", name); + + return t; +} diff --git a/src/external/samurai/tree.c b/src/external/samurai/tree.c new file mode 100644 index 000000000..a342a145d --- /dev/null +++ b/src/external/samurai/tree.c @@ -0,0 +1,130 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-FileCopyrightText: 2005-2014 Rich Felker, et al. + * SPDX-License-Identifier: MIT + * + * Based on musl's src/search/tsearch.c, by Szabolcs Nagy. + */ + +#include "compat.h" + +#include +#include + +#include "external/samurai/ctx.h" + +#include "external/samurai/tree.h" +#include "external/samurai/util.h" + +#include "external/samurai/ctx.h" + +#define MAXH (sizeof(void *) * 8 * 3 / 2) + +static inline int +samu_height(struct samu_treenode *n) +{ + return n ? n->height : 0; +} + +static int +samu_rot(struct samu_treenode **p, struct samu_treenode *x, int dir /* deeper side */) +{ + struct samu_treenode *y = x->child[dir]; + struct samu_treenode *z = y->child[!dir]; + int hx = x->height; + int hz = samu_height(z); + + if (hz > samu_height(y->child[dir])) { + /* + * x + * / \ dir z + * A y / \ + * / \ --> x y + * z D /| |\ + * / \ A B C D + * B C + */ + x->child[dir] = z->child[!dir]; + y->child[!dir] = z->child[dir]; + z->child[!dir] = x; + z->child[dir] = y; + x->height = hz; + y->height = hz; + z->height = hz + 1; + } else { + /* + * x y + * / \ / \ + * A y --> x D + * / \ / \ + * z D A z + */ + x->child[dir] = z; + y->child[!dir] = x; + x->height = hz + 1; + y->height = hz + 2; + z = y; + } + *p = z; + return z->height - hx; +} + +static int +samu_balance(struct samu_treenode **p) +{ + struct samu_treenode *n = *p; + int h0 = samu_height(n->child[0]); + int h1 = samu_height(n->child[1]); + + if (h0 - h1 + 1u < 3u) { + int old = n->height; + n->height = h0 < h1 ? h1 + 1 : h0 + 1; + return n->height - old; + } + return samu_rot(p, n, h0 < h1); +} + +struct samu_treenode * +samu_treefind(struct samu_treenode *n, const char *key) +{ + int c; + + while (n) { + c = strcmp(key, n->key); + if (c == 0) + return n; + n = n->child[c > 0]; + } + return NULL; +} + +void * +samu_treeinsert(struct samu_ctx *ctx, struct samu_treenode **rootp, char *key, void *value) +{ + struct samu_treenode **a[MAXH], *n = *rootp, *r; + void *old; + int i = 0, c; + + a[i++] = rootp; + while (n) { + c = strcmp(key, n->key); + if (c == 0) { + old = n->value; + n->value = value; + return old; + } + a[i++] = &n->child[c > 0]; + n = n->child[c > 0]; + } + r = samu_xmalloc(&ctx->arena, sizeof(*r)); + r->key = key; + r->value = value; + r->child[0] = r->child[1] = NULL; + r->height = 1; + /* insert new node, rebalance ancestors. */ + *a[--i] = r; + while (i && samu_balance(a[--i])) + ; + return NULL; +} diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c new file mode 100644 index 000000000..0755a2ee1 --- /dev/null +++ b/src/external/samurai/util.c @@ -0,0 +1,314 @@ +/* + * SPDX-FileCopyrightText: Michael Forney + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: MIT + */ + +#include "compat.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "assert.h" +#include "buf_size.h" +#include "external/samurai/ctx.h" +#include "log.h" +#include "error.h" +#include "platform/mem.h" + +#include "external/samurai/util.h" + +static void +samu_vwarn(const char *fmt, va_list ap) +{ + fprintf(stderr, "samu: "); + vfprintf(stderr, fmt, ap); + if (fmt[0] && fmt[strlen(fmt) - 1] == ':') { + putc(' ', stderr); + perror(NULL); + } else { + putc('\n', stderr); + } +} + +void +samu_warn(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + samu_vwarn(fmt, ap); + va_end(ap); +} + +void +samu_fatal(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + samu_vwarn(fmt, ap); + va_end(ap); + exit(1); +} + +void * +samu_xmalloc(struct samu_arena *a, size_t n) +{ + return samu_arena_alloc(a, n); +} + +static void +samu_arena_push_block(struct samu_arena *a, uint64_t size) +{ + ++a->blocks_len; + a->blocks = z_realloc(a->blocks, sizeof(const char *) * a->blocks_len); + a->allocd += size; + a->blocks[a->blocks_len - 1] = z_calloc(1, size); +} + +#define SAMU_ARENA_DEF_BLOCK_SIZE BUF_SIZE_1m + +void +samu_arena_init(struct samu_arena *a) +{ + samu_arena_push_block(a, SAMU_ARENA_DEF_BLOCK_SIZE); +} + +void +samu_arena_destroy(struct samu_arena *a) +{ + size_t i; + for (i = 0; i < a->blocks_len; ++i) { + z_free(a->blocks[i]); + } + + L("samu allocd %zd blocks, a:%zd, f:%zd, r:%3.3f\n", a->blocks_len, a->allocd, a->filled, (float)a->filled / (float)a->allocd * 100.0f); + + z_free(a->blocks); +} + +void * +samu_arena_alloc(struct samu_arena *a, uint64_t size) +{ + uint64_t align = -a->i & 7; + a->i += align; + + if (a->i + size > SAMU_ARENA_DEF_BLOCK_SIZE || size > SAMU_ARENA_DEF_BLOCK_SIZE) { + uint64_t new_block_size = size > SAMU_ARENA_DEF_BLOCK_SIZE ? size : SAMU_ARENA_DEF_BLOCK_SIZE; + + samu_arena_push_block(a, new_block_size); + a->i = 0; + } + + a->filled += size; + + char *mem = a->blocks[a->blocks_len - 1] + a->i; + a->i += align + size; + return mem; +} + + +void * +samu_arena_realloc(struct samu_arena *a, void *p, size_t old, size_t new) +{ + char *mem = samu_arena_alloc(a, new); + memcpy(mem, p, old); + return mem; +} + +static void * +samu_reallocarray(struct samu_arena *a, void *p, size_t old, size_t new, size_t m) +{ + if (m && new > SIZE_MAX / m) { + errno = ENOMEM; + return NULL; + } + return samu_arena_realloc(a, p, old * m, new * m); +} + +void * +samu_xreallocarray(struct samu_arena *a, void *p, size_t old, size_t new, size_t m) +{ + return samu_reallocarray(a, p, old, new, m); +} + +char * +samu_xmemdup(struct samu_arena *a, const char *s, size_t n) +{ + char *p; + + p = samu_xmalloc(a, n); + memcpy(p, s, n); + + return p; +} + +int +samu_xasprintf(struct samu_arena *a, char **s, const char *fmt, ...) +{ + va_list ap; + int ret; + size_t n; + + va_start(ap, fmt); + ret = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + assert(!(ret < 0)); + n = ret + 1; + *s = samu_xmalloc(a, n); + va_start(ap, fmt); + ret = vsnprintf(*s, n, fmt, ap); + va_end(ap); + assert(ret < 0 || (size_t)ret >= n); + assert(!(ret < 0 || (size_t)ret >= n)); + + return ret; +} + +void +samu_bufadd(struct samu_arena *a, struct samu_buffer *buf, char c) +{ + if (buf->len >= buf->cap) { + size_t newcap = buf->cap ? buf->cap * 2 : 1 << 8; + buf->data = samu_arena_realloc(a, buf->data, buf->cap, newcap); + buf->cap = newcap; + } + buf->data[buf->len++] = c; +} + +struct samu_string * +samu_mkstr(struct samu_arena *a, size_t n) +{ + struct samu_string *str; + + str = samu_xmalloc(a, sizeof(*str) + n + 1); + str->n = n; + + return str; +} + +void +samu_delevalstr(void *ptr) +{ +} + +void +samu_canonpath(struct samu_string *path) +{ + char *component[60]; + int n; + char *s, *d, *end; + + if (path->n == 0) + samu_fatal("empty path"); + s = d = path->s; + end = path->s + path->n; + n = 0; + if (*s == '/') { + ++s; + ++d; + } + while (s < end) { + switch (s[0]) { + case '/': + ++s; + continue; + case '.': + switch (s[1]) { + case '\0': case '/': + s += 2; + continue; + case '.': + if (s[2] != '/' && s[2] != '\0') + break; + if (n > 0) { + d = component[--n]; + } else { + *d++ = s[0]; + *d++ = s[1]; + *d++ = s[2]; + } + s += 3; + continue; + } + } + if (n == ARRAY_LEN(component)) + samu_fatal("path has too many components: %s", path->s); + component[n++] = d; + while (*s != '/' && *s != '\0') + *d++ = *s++; + *d++ = *s++; + } + if (d == path->s) { + *d++ = '.'; + *d = '\0'; + } else { + *--d = '\0'; + } + path->n = d - path->s; +} + +int +samu_makedirs(struct samu_string *path, bool parent) +{ + int ret; + struct stat st; + char *s, *end; + + ret = 0; + end = path->s + path->n; + for (s = end - parent; s > path->s; --s) { + if (*s != '/' && *s) + continue; + *s = '\0'; + if (stat(path->s, &st) == 0) + break; + if (errno != ENOENT) { + samu_warn("stat %s:", path->s); + ret = -1; + break; + } + } + if (s > path->s && s < end) + *s = '/'; + while (++s <= end - parent) { + if (*s != '\0') + continue; + if (ret == 0 && mkdir(path->s, 0777) < 0 && errno != EEXIST) { + samu_warn("mkdir %s:", path->s); + ret = -1; + } + if (s < end) + *s = '/'; + } + + return ret; +} + +int +samu_writefile(const char *name, struct samu_string *s) +{ + FILE *f; + int ret; + + f = fopen(name, "w"); + if (!f) { + samu_warn("open %s:", name); + return -1; + } + ret = 0; + if (s && (fwrite(s->s, 1, s->n, f) != s->n || fflush(f) != 0)) { + samu_warn("write %s:", name); + ret = -1; + } + fclose(f); + + return ret; +} From c9a164871ec315a26e5054891b2f980554b0496b Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Fri, 5 Jan 2024 20:16:53 -0500 Subject: [PATCH 03/52] factor samu output to allow outputting to file This allows muon to run samu -t compdb internally and redirect the output to compile_commands.json without the ugly fs_redirect functions. --- include/backend/ninja.h | 2 +- include/external/samurai.h | 9 +- include/external/samurai/ctx.h | 2 + include/external/samurai/samu.h | 9 -- include/external/samurai/util.h | 13 ++- include/platform/filesystem.h | 2 - src/backend/ninja.c | 29 ++---- src/cmd_test.c | 2 +- src/external/samurai.c | 10 --- src/external/samurai/build.c | 4 +- src/external/samurai/samu.c | 50 ++++++----- src/external/samurai/tool.c | 151 ++++++++++++++++++-------------- src/external/samurai/util.c | 35 ++++++++ src/external/samurai_null.c | 2 +- src/main.c | 2 +- src/platform/filesystem.c | 45 ---------- 16 files changed, 180 insertions(+), 187 deletions(-) delete mode 100644 include/external/samurai/samu.h diff --git a/include/backend/ninja.h b/include/backend/ninja.h index ce103ab9c..7a4d35fb4 100644 --- a/include/backend/ninja.h +++ b/include/backend/ninja.h @@ -15,5 +15,5 @@ struct write_tgt_ctx { }; bool ninja_write_all(struct workspace *wk); -int ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture); +bool ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture); #endif diff --git a/include/external/samurai.h b/include/external/samurai.h index bfe5f265d..32e21fd00 100644 --- a/include/external/samurai.h +++ b/include/external/samurai.h @@ -6,10 +6,15 @@ #ifndef MUON_EXTERNAL_SAMURAI_H #define MUON_EXTERNAL_SAMURAI_H -#include #include +#include +#include + +struct samu_opts { + FILE *out; +}; extern const bool have_samurai; -bool muon_samu(uint32_t argc, char *const argv[]); +bool samu_main(int argc, char *argv[], struct samu_opts *opts); #endif diff --git a/include/external/samurai/ctx.h b/include/external/samurai/ctx.h index 25a4bc5ed..5192f8074 100644 --- a/include/external/samurai/ctx.h +++ b/include/external/samurai/ctx.h @@ -212,6 +212,8 @@ struct samu_ctx { struct samu_rule phonyrule; struct samu_pool consolepool; struct samu_arena arena; + + FILE *out; }; struct samu_tool { diff --git a/include/external/samurai/samu.h b/include/external/samurai/samu.h deleted file mode 100644 index 0999fe554..000000000 --- a/include/external/samurai/samu.h +++ /dev/null @@ -1,9 +0,0 @@ -/* - * SPDX-FileCopyrightText: Stone Tickle - * SPDX-License-Identifier: GPL-3.0-only - */ - -#ifndef MUON_EXTERNAL_SAMU_SAMU_H -#define MUON_EXTERNAL_SAMU_SAMU_H -int samu_main(int argc, char *argv[]); -#endif diff --git a/include/external/samurai/util.h b/include/external/samurai/util.h index 0799e640c..e2afb8a3b 100644 --- a/include/external/samurai/util.h +++ b/include/external/samurai/util.h @@ -6,8 +6,17 @@ #ifndef MUON_EXTERNAL_SAMU_UTIL_H #define MUON_EXTERNAL_SAMU_UTIL_H -void samu_warn(const char *, ...); -void samu_fatal(const char *, ...); +void samu_warn(const char *, ...) +MUON_ATTR_FORMAT(printf, 1, 2); +void samu_fatal(const char *, ...) +MUON_ATTR_FORMAT(printf, 1, 2); + +int samu_vprintf(struct samu_ctx *ctx, const char *fmt, va_list ap); +int samu_printf(struct samu_ctx *ctx, const char *fmt, ...) +MUON_ATTR_FORMAT(printf, 2, 3); +void samu_puts(struct samu_ctx *ctx, const char *str); +void samu_puts_no_newline(struct samu_ctx *ctx, const char *str); +void samu_putchar(struct samu_ctx *ctx, const char c); void samu_arena_init(struct samu_arena *a); void samu_arena_destroy(struct samu_arena *a); diff --git a/include/platform/filesystem.h b/include/platform/filesystem.h index 7bb331fc7..7058cbc9a 100644 --- a/include/platform/filesystem.h +++ b/include/platform/filesystem.h @@ -51,8 +51,6 @@ bool fs_find_cmd(struct workspace *wk, struct sbuf *buf, const char *cmd); bool fs_has_cmd(const char *cmd); void fs_source_destroy(struct source *src); void fs_source_dup(const struct source *src, struct source *dup); -bool fs_redirect(const char *path, const char *mode, int fd, int *old_fd); -bool fs_redirect_restore(int fd, int old_fd); bool fs_copy_file(const char *src, const char *dest); bool fs_copy_dir(const char *src_base, const char *dest_base); bool fs_fileno(FILE *f, int *ret); diff --git a/src/backend/ninja.c b/src/backend/ninja.c index ff7d2204c..75b30714f 100644 --- a/src/backend/ninja.c +++ b/src/backend/ninja.c @@ -262,7 +262,7 @@ ninja_write_all(struct workspace *wk) obj_array_push(wk, compdb_args, make_str(wk, "-t")); obj_array_push(wk, compdb_args, make_str(wk, "compdb")); obj_array_extend_nodup(wk, compdb_args, ctx.compiler_rule_arr); - if (ninja_run(wk, compdb_args, wk->build_root, "compile_commands.json") != 0) { + if (!ninja_run(wk, compdb_args, wk->build_root, "compile_commands.json")) { LOG_E("error writing compile_commands.json"); } @@ -272,13 +272,13 @@ ninja_write_all(struct workspace *wk) return true; } -int +bool ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture) { const char *argstr; uint32_t argstr_argc; - int ret = 1; + bool ret = false; char *const *argv = NULL; uint32_t argc; SBUF_manual(cwd); @@ -291,35 +291,24 @@ ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture } } - bool have_stdout_fileno; - int stdout_fileno; - if (capture) { - have_stdout_fileno = fs_fileno(stdout, &stdout_fileno); - } else { - have_stdout_fileno = true; - } - - if (have_samurai && have_stdout_fileno) { + if (have_samurai) { join_args_argstr(wk, &argstr, &argstr_argc, args); argc = argstr_to_argv(argstr, argstr_argc, "samu", &argv); - int old_stdout; - + struct samu_opts samu_opts = { .out = stdout }; if (capture) { - if (!fs_redirect(capture, "wb", stdout_fileno, &old_stdout)) { + if (!(samu_opts.out = fs_fopen(capture, "wb"))) { goto ret; } } - bool res = muon_samu(argc, argv); + ret = samu_main(argc, (char **)argv, &samu_opts); if (capture) { - if (!fs_redirect_restore(stdout_fileno, old_stdout)) { + if (!fs_fclose(samu_opts.out)) { goto ret; } } - - ret = res ? 0 : 1; } else { struct run_cmd_ctx cmd_ctx = { 0 }; @@ -367,7 +356,7 @@ ninja_run(struct workspace *wk, obj args, const char *chdir, const char *capture } } - ret = cmd_ctx.status; + ret = cmd_ctx.status == 0; run_cmd_done: sbuf_destroy(&cmd); run_cmd_ctx_destroy(&cmd_ctx); diff --git a/src/cmd_test.c b/src/cmd_test.c index 42ce1133f..204e170ca 100644 --- a/src/cmd_test.c +++ b/src/cmd_test.c @@ -855,7 +855,7 @@ run_project_tests(struct workspace *wk, void *_ctx, obj proj_name, obj arr) if (get_obj_array(wk, ctx->deps)->len && !ctx->opts->no_rebuild) { obj ninja_cmd; obj_array_dedup(wk, ctx->deps, &ninja_cmd); - if (ninja_run(wk, ninja_cmd, NULL, NULL) != 0) { + if (!ninja_run(wk, ninja_cmd, NULL, NULL)) { LOG_W("failed to run ninja"); } } diff --git a/src/external/samurai.c b/src/external/samurai.c index 221df7853..f14e2804b 100644 --- a/src/external/samurai.c +++ b/src/external/samurai.c @@ -5,16 +5,6 @@ #include "compat.h" -#include "buf_size.h" #include "external/samurai.h" -#include "external/samurai/samu.h" -#include "platform/filesystem.h" -#include "platform/path.h" const bool have_samurai = true; - -bool -muon_samu(uint32_t argc, char *const argv[]) -{ - return samu_main(argc, (char **)argv) == 0; -} diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index b1ed754e8..371e53e51 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -268,8 +268,8 @@ samu_printstatus(struct samu_ctx *ctx, struct samu_edge *e, struct samu_string * if (!description || description->n == 0) description = cmd; samu_formatstatus(ctx, status, sizeof(status)); - fputs(status, stdout); - puts(description->s); + samu_puts_no_newline(ctx, status); + samu_puts(ctx, description->s); } static int diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c index 5026b2e78..5d86d55af 100644 --- a/src/external/samurai/samu.c +++ b/src/external/samurai/samu.c @@ -13,9 +13,11 @@ #include #include /* for chdir */ +#include "assert.h" #include "buf_size.h" #include "external/samurai/ctx.h" +#include "external/samurai.h" #include "external/samurai/arg.h" #include "external/samurai/build.h" #include "external/samurai/deps.h" @@ -23,7 +25,6 @@ #include "external/samurai/graph.h" #include "external/samurai/log.h" #include "external/samurai/parse.h" -#include "external/samurai/samu.h" #include "external/samurai/tool.h" #include "external/samurai/util.h" @@ -135,19 +136,28 @@ samu_parseenvargs(struct samu_ctx *ctx, char *env) } SAMU_ARGEND } -static const char * -samu_progname(const char *arg, const char *def) -{ - const char *slash; +static void +samu_init_ctx(struct samu_ctx *ctx, struct samu_opts *opts) { + *ctx = (struct samu_ctx){ + .buildopts = {.maxfail = 1}, + .phonyrule = {.name = "phony"}, + .consolepool = {.name = "console", .maxjobs = 1}, + .out = stdout, + }; + + if (opts) { + if (ctx->out) { + ctx->out = opts->out; + } + } - if (!arg) - return def; - slash = strrchr(arg, '/'); - return slash ? slash + 1 : arg; + ctx->argv0 = ""; + + samu_arena_init(&ctx->arena); } -int -samu_main(int argc, char *argv[]) +bool +samu_main(int argc, char *argv[], struct samu_opts *opts) { char *builddir, *manifest = "build.ninja", *end, *arg; const struct samu_tool *tool = NULL; @@ -155,22 +165,16 @@ samu_main(int argc, char *argv[]) long num; int tries; - struct samu_ctx samu_ctx = { - .buildopts = {.maxfail = 1}, - .phonyrule = {.name = "phony"}, - .consolepool = {.name = "console", .maxjobs = 1}, - }, *ctx = &samu_ctx; - - samu_arena_init(&ctx->arena); + struct samu_ctx _ctx, *ctx = &_ctx; + samu_init_ctx(ctx, opts); - ctx->argv0 = samu_progname(argv[0], "samu"); samu_parseenvargs(ctx, getenv("SAMUFLAGS")); SAMU_ARGBEGIN { case '-': arg = SAMU_EARGF(samu_usage(ctx)); if (strcmp(arg, "version") == 0) { - printf("%d.%d.0\n", ninjamajor, ninjaminor); - return 0; + samu_printf(ctx, "%d.%d.0\n", ninjamajor, ninjaminor); + return true; } else if (strcmp(arg, "verbose") == 0) { ctx->buildopts.verbose = true; } else { @@ -255,7 +259,7 @@ samu_main(int argc, char *argv[]) if (tool) { int r = tool->run(ctx, argc, argv); samu_arena_destroy(&ctx->arena); - return r; + return r == 0; } /* load the build log */ @@ -296,5 +300,5 @@ samu_main(int argc, char *argv[]) samu_depsclose(ctx); samu_arena_destroy(&ctx->arena); - return 0; + return true; } diff --git a/src/external/samurai/tool.c b/src/external/samurai/tool.c index 29da424fb..b9c9e4911 100644 --- a/src/external/samurai/tool.c +++ b/src/external/samurai/tool.c @@ -25,11 +25,11 @@ #include "external/samurai/util.h" static int -samu_cleanpath(struct samu_string *path) +samu_cleanpath(struct samu_ctx *ctx, struct samu_string *path) { if (path) { if (remove(path->s) == 0) { - printf("remove %s\n", path->s); + samu_printf(ctx, "remove %s\n", path->s); } else if (errno != ENOENT) { samu_warn("remove %s:", path->s); return -1; @@ -46,12 +46,12 @@ samu_cleanedge(struct samu_ctx *ctx, struct samu_edge *e) size_t i; for (i = 0; i < e->nout; ++i) { - if (samu_cleanpath(e->out[i]->path) < 0) + if (samu_cleanpath(ctx, e->out[i]->path) < 0) ret = -1; } - if (samu_cleanpath(samu_edgevar(ctx, e, "rspfile", false)) < 0) + if (samu_cleanpath(ctx, samu_edgevar(ctx, e, "rspfile", false)) < 0) ret = -1; - if (samu_cleanpath(samu_edgevar(ctx, e, "depfile", false)) < 0) + if (samu_cleanpath(ctx, samu_edgevar(ctx, e, "depfile", false)) < 0) ret = -1; return ret; @@ -65,7 +65,7 @@ samu_cleantarget(struct samu_ctx *ctx, struct samu_node *n) if (!n->gen || n->gen->rule == &ctx->phonyrule) return 0; - if (samu_cleanpath(n->path) < 0) + if (samu_cleanpath(ctx, n->path) < 0) ret = -1; for (i = 0; i < n->gen->nin; ++i) { if (samu_cleantarget(ctx, n->gen->in[i]) < 0) @@ -153,7 +153,7 @@ samu_targetcommands(struct samu_ctx *ctx, struct samu_node *n) samu_targetcommands(ctx, e->in[i]); command = samu_edgevar(ctx, e, "command", true); if (command && command->n) - puts(command->s); + samu_puts(ctx, command->s); } static int @@ -179,7 +179,7 @@ samu_commands(struct samu_ctx *ctx, int argc, char *argv[]) } static void -samu_printjson(const char *s, size_t n, bool join) +samu_printjson(struct samu_ctx *ctx, const char *s, size_t n, bool join) { size_t i; char c; @@ -189,7 +189,7 @@ samu_printjson(const char *s, size_t n, bool join) switch (c) { case '"': case '\\': - putchar('\\'); + samu_putchar(ctx, '\\'); break; case '\n': if (join) @@ -198,7 +198,7 @@ samu_printjson(const char *s, size_t n, bool join) case '\0': return; } - putchar(c); + samu_putchar(ctx, c); } } @@ -211,60 +211,75 @@ samu_compdb(struct samu_ctx *ctx, int argc, char *argv[]) bool expandrsp = false, first = true; int i; size_t off; + bool all = false; SAMU_ARGBEGIN { case 'x': expandrsp = true; break; + case 'a': + all = true; + break; default: - fprintf(stderr, "usage: %s ... -t compdb [-x] [rules...]\n", ctx->argv0); + fprintf(stderr, "usage: %s ... -t compdb [-xa] [rules...]\n", ctx->argv0); return 2; } SAMU_ARGEND if (!getcwd(dir, sizeof(dir))) samu_fatal("getcwd:"); - putchar('['); + samu_putchar(ctx, '['); for (e = ctx->graph.alledges; e; e = e->allnext) { if (e->nin == 0) continue; - for (i = 0; i < argc; ++i) { - if (strcmp(e->rule->name, argv[i]) == 0) { - if (first) - first = false; - else - putchar(','); - - printf("\n {\n \"directory\": \""); - samu_printjson(dir, -1, false); - - printf("\",\n \"command\": \""); - cmd = samu_edgevar(ctx, e, "command", true); - rspfile = expandrsp ? samu_edgevar(ctx, e, "rspfile", true) : NULL; - p = rspfile ? strstr(cmd->s, rspfile->s) : NULL; - if (!p || p == cmd->s || p[-1] != '@') { - samu_printjson(cmd->s, cmd->n, false); - } else { - off = p - cmd->s; - samu_printjson(cmd->s, off - 1, false); - content = samu_edgevar(ctx, e, "rspfile_content", true); - samu_printjson(content->s, content->n, true); - off += rspfile->n; - samu_printjson(cmd->s + off, cmd->n - off, false); - } - - printf("\",\n \"file\": \""); - samu_printjson(e->in[0]->path->s, -1, false); + if (e->rule == &ctx->phonyrule) { + continue; + } - printf("\",\n \"output\": \""); - samu_printjson(e->out[0]->path->s, -1, false); + if (!all) { + for (i = 0; i < argc; ++i) { + if (strcmp(e->rule->name, argv[i]) == 0) { + break; + } + } - printf("\"\n }"); - break; + if (i == argc) { + continue; } } + + if (first) + first = false; + else + samu_putchar(ctx, ','); + + samu_printf(ctx, "\n {\n \"directory\": \""); + samu_printjson(ctx, dir, -1, false); + + samu_printf(ctx, "\",\n \"command\": \""); + cmd = samu_edgevar(ctx, e, "command", true); + rspfile = expandrsp ? samu_edgevar(ctx, e, "rspfile", true) : NULL; + p = rspfile ? strstr(cmd->s, rspfile->s) : NULL; + if (!p || p == cmd->s || p[-1] != '@') { + samu_printjson(ctx, cmd->s, cmd->n, false); + } else { + off = p - cmd->s; + samu_printjson(ctx, cmd->s, off - 1, false); + content = samu_edgevar(ctx, e, "rspfile_content", true); + samu_printjson(ctx, content->s, content->n, true); + off += rspfile->n; + samu_printjson(ctx, cmd->s + off, cmd->n - off, false); + } + + samu_printf(ctx, "\",\n \"file\": \""); + samu_printjson(ctx, e->in[0]->path->s, -1, false); + + samu_printf(ctx, "\",\n \"output\": \""); + samu_printjson(ctx, e->out[0]->path->s, -1, false); + + samu_printf(ctx, "\"\n }"); } - puts("\n]"); + samu_puts(ctx, "\n]"); if (fflush(stdout) || ferror(stdout)) samu_fatal("write failed"); @@ -279,7 +294,7 @@ samu_graphnode(struct samu_ctx *ctx, struct samu_node *n) size_t i; const char *style; - printf("\"%p\" [label=\"%s\"]\n", (void *)n, n->path->s); + samu_printf(ctx, "\"%p\" [label=\"%s\"]\n", (void *)n, n->path->s); if (!e || (e->flags & FLAG_WORK)) return; @@ -289,14 +304,14 @@ samu_graphnode(struct samu_ctx *ctx, struct samu_node *n) samu_graphnode(ctx, e->in[i]); if (e->nin == 1 && e->nout == 1) { - printf("\"%p\" -> \"%p\" [label=\"%s\"]\n", (void *)e->in[0], (void *)e->out[0], e->rule->name); + samu_printf(ctx, "\"%p\" -> \"%p\" [label=\"%s\"]\n", (void *)e->in[0], (void *)e->out[0], e->rule->name); } else { - printf("\"%p\" [label=\"%s\", shape=ellipse]\n", (void *)e, e->rule->name); + samu_printf(ctx, "\"%p\" [label=\"%s\", shape=ellipse]\n", (void *)e, e->rule->name); for (i = 0; i < e->nout; ++i) - printf("\"%p\" -> \"%p\"\n", (void *)e, (void *)e->out[i]); + samu_printf(ctx, "\"%p\" -> \"%p\"\n", (void *)e, (void *)e->out[i]); for (i = 0; i < e->nin; ++i) { style = i >= e->inorderidx ? " style=dotted" : ""; - printf("\"%p\" -> \"%p\" [arrowhead=none%s]\n", (void *)e->in[i], (void *)e, style); + samu_printf(ctx, "\"%p\" -> \"%p\" [arrowhead=none%s]\n", (void *)e->in[i], (void *)e, style); } } } @@ -306,10 +321,10 @@ samu_graph(struct samu_ctx *ctx, int argc, char *argv[]) { struct samu_node *n; - puts("digraph ninja {"); - puts("rankdir=\"LR\""); - puts("node [fontsize=10, shape=box, height=0.25]"); - puts("edge [fontsize=10]"); + samu_puts(ctx, "digraph ninja {"); + samu_puts(ctx, "rankdir=\"LR\""); + samu_puts(ctx, "node [fontsize=10, shape=box, height=0.25]"); + samu_puts(ctx, "edge [fontsize=10]"); if (argc > 1) { while (*++argv) { @@ -322,7 +337,7 @@ samu_graph(struct samu_ctx *ctx, int argc, char *argv[]) samu_defaultnodes(ctx, samu_graphnode); } - puts("}"); + samu_puts(ctx, "}"); if (fflush(stdout) || ferror(stdout)) samu_fatal("write failed"); @@ -348,18 +363,18 @@ samu_query(struct samu_ctx *ctx, int argc, char *argv[]) n = samu_nodeget(ctx, path, 0); if (!n) samu_fatal("unknown target '%s'", path); - printf("%s:\n", argv[i]); + samu_printf(ctx, "%s:\n", argv[i]); e = n->gen; if (e) { - printf(" input: %s\n", e->rule->name); + samu_printf(ctx, " input: %s\n", e->rule->name); for (j = 0; j < e->nin; ++j) - printf(" %s\n", e->in[j]->path->s); + samu_printf(ctx, " %s\n", e->in[j]->path->s); } - puts(" outputs:"); + samu_puts(ctx, " outputs:"); for (j = 0; j < n->nuse; ++j) { e = n->use[j]; for (k = 0; k < e->nout; ++k) - printf(" %s\n", e->out[k]->path->s); + samu_printf(ctx, " %s\n", e->out[k]->path->s); } } @@ -367,21 +382,21 @@ samu_query(struct samu_ctx *ctx, int argc, char *argv[]) } static void -samu_targetsdepth(struct samu_node *n, size_t depth, size_t indent) +samu_targetsdepth(struct samu_ctx *ctx, struct samu_node *n, size_t depth, size_t indent) { struct samu_edge *e = n->gen; size_t i; for (i = 0; i < indent; ++i) - printf(" "); + samu_printf(ctx, " "); if (e) { - printf("%s: %s\n", n->path->s, e->rule->name); + samu_printf(ctx, "%s: %s\n", n->path->s, e->rule->name); if (depth != 1) { for (i = 0; i < e->nin; ++i) - samu_targetsdepth(e->in[i], depth - 1, indent + 1); + samu_targetsdepth(ctx, e->in[i], depth - 1, indent + 1); } } else { - puts(n->path->s); + samu_puts(ctx, n->path->s); } } @@ -415,7 +430,7 @@ samu_targets(struct samu_ctx *ctx, int argc, char *argv[]) for (e = ctx->graph.alledges; e; e = e->allnext) { for (i = 0; i < e->nout; ++i) { if (e->out[i]->nuse == 0) - samu_targetsdepth(e->out[i], depth, 0); + samu_targetsdepth(ctx, e->out[i], depth, 0); } } } else if (strcmp(mode, "rule") == 0) { @@ -424,17 +439,17 @@ samu_targets(struct samu_ctx *ctx, int argc, char *argv[]) if (!name) { for (i = 0; i < e->nin; ++i) { if (!e->in[i]->gen) - puts(e->in[i]->path->s); + samu_puts(ctx, e->in[i]->path->s); } } else if (strcmp(e->rule->name, name) == 0) { for (i = 0; i < e->nout; ++i) - puts(e->out[i]->path->s); + samu_puts(ctx, e->out[i]->path->s); } } } else if (strcmp(mode, "all") == 0 && argc == 2) { for (e = ctx->graph.alledges; e; e = e->allnext) { for (i = 0; i < e->nout; ++i) - printf("%s: %s\n", e->out[i]->path->s, e->rule->name); + samu_printf(ctx, "%s: %s\n", e->out[i]->path->s, e->rule->name); } } else { samu_targetsusage(ctx); diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index 0755a2ee1..ddcb32e6c 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -58,6 +58,41 @@ samu_fatal(const char *fmt, ...) exit(1); } +int +samu_vprintf(struct samu_ctx *ctx, const char *fmt, va_list ap) +{ + return vfprintf(ctx->out, fmt, ap); +} + +int +samu_printf(struct samu_ctx *ctx, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + int r = samu_vprintf(ctx, fmt, ap); + va_end(ap); + return r; +} + +void +samu_puts_no_newline(struct samu_ctx *ctx, const char *str) +{ + fputs(str, ctx->out); +} + +void +samu_puts(struct samu_ctx *ctx, const char *str) +{ + fputs(str, ctx->out); + fputc('\n', ctx->out); +} + +void +samu_putchar(struct samu_ctx *ctx, const char c) +{ + fputc(c, ctx->out); +} + void * samu_xmalloc(struct samu_arena *a, size_t n) { diff --git a/src/external/samurai_null.c b/src/external/samurai_null.c index 345bf2627..81ab8da3a 100644 --- a/src/external/samurai_null.c +++ b/src/external/samurai_null.c @@ -11,7 +11,7 @@ const bool have_samurai = false; bool -muon_samu(uint32_t argc, char *const argv[]) +samu_main(int argc, char *argv[], struct samu_opts *opts) { LOG_W("samurai not enabled"); return false; diff --git a/src/main.c b/src/main.c index 7a0bfb85a..f74cca293 100644 --- a/src/main.c +++ b/src/main.c @@ -679,7 +679,7 @@ cmd_internal(uint32_t argc, uint32_t argi, char *const argv[]) static bool cmd_samu(uint32_t argc, uint32_t argi, char *const argv[]) { - return muon_samu(argc - argi, (char **)&argv[argi]); + return samu_main(argc - argi, (char **)&argv[argi], 0); } static bool diff --git a/src/platform/filesystem.c b/src/platform/filesystem.c index 99556a19b..b7b5f165d 100644 --- a/src/platform/filesystem.c +++ b/src/platform/filesystem.c @@ -350,17 +350,6 @@ fs_has_cmd(const char *cmd) return res; } -static bool -fs_dup2(int oldfd, int newfd) -{ - if ((dup2(oldfd, newfd)) == -1) { - LOG_E("failed dup2(%d, %d): %s", oldfd, newfd, strerror(errno)); - return false; - } - - return true; -} - bool fs_fileno(FILE *f, int *ret) { @@ -375,40 +364,6 @@ fs_fileno(FILE *f, int *ret) return true; } -bool -fs_redirect(const char *path, const char *mode, int fd, int *old_fd) -{ - FILE *out; - int out_fd; - - if ((*old_fd = dup(fd)) == -1) { - LOG_E("failed dup(%d): %s", fd, strerror(errno)); - return false; - } else if (!(out = fs_fopen(path, mode))) { - return false; - } else if (!fs_fileno(out, &out_fd)) { - return false; - } else if (!fs_dup2(out_fd, fd)) { - return false; - } else if (!fs_fclose(out)) { - return false; - } - - return true; -} - -bool -fs_redirect_restore(int fd, int old_fd) -{ - if (!fs_dup2(old_fd, fd)) { - return false; - } else if (close(old_fd) == -1) { - LOG_E("failed close(%d): %s", old_fd, strerror(errno)); - return false; - } - return true; -} - struct fs_copy_dir_ctx { const char *src_base, *dest_base; }; From 9deaf33e34fe87c83a6a9ec40647cf9d5874ffee Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Fri, 5 Jan 2024 20:17:06 -0500 Subject: [PATCH 04/52] error on external_program.full_path if multi elem --- src/functions/external_program.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/functions/external_program.c b/src/functions/external_program.c index d841d01c1..4d0a0716c 100644 --- a/src/functions/external_program.c +++ b/src/functions/external_program.c @@ -54,6 +54,12 @@ func_external_program_path(struct workspace *wk, obj rcvr, uint32_t args_node, o return false; } + struct obj_external_program *ep = get_obj_external_program(wk, rcvr); + if (get_obj_array(wk, ep->cmd_array)->len > 1) { + interp_error(wk, args_node, "cannot return the full_path() of an external program with multiple elements (have: %o)\n", ep->cmd_array); + return false; + } + obj_array_index(wk, get_obj_external_program(wk, rcvr)->cmd_array, 0, res); return true; } From c3a525b4665d52be36acad7f5443979174c564e6 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Fri, 5 Jan 2024 20:50:21 -0500 Subject: [PATCH 05/52] remove ninja requirement from muon bootstrap --- .builds/alpine.yml | 8 ++----- .builds/netbsd.yml | 6 +---- README.md | 13 +++-------- meson_options.txt | 2 +- src/amalgam.c | 34 +++++++++++++++------------ subprojects/samurai.wrap | 7 ------ tools/bootstrap_ninja.sh | 35 +--------------------------- tools/ci/bootstrap.sh | 11 +++++---- tools/ci/fullbootstrap.sh | 11 --------- tools/ci/git_archive_with_samurai.sh | 14 ----------- tools/ci/solaris11.sh | 5 ++-- 11 files changed, 35 insertions(+), 111 deletions(-) delete mode 100644 subprojects/samurai.wrap delete mode 100755 tools/ci/fullbootstrap.sh delete mode 100755 tools/ci/git_archive_with_samurai.sh diff --git a/.builds/alpine.yml b/.builds/alpine.yml index 85b863a8b..3d42c835c 100644 --- a/.builds/alpine.yml +++ b/.builds/alpine.yml @@ -66,7 +66,7 @@ tasks: fi - build_gcc: | cd muon - CC=gcc tools/ci/fullbootstrap.sh -Dbuildtype=release -Dstatic=true -Dwebsite=true + CC=gcc tools/ci/bootstrap.sh build -Dbuildtype=release -Dstatic=true -Dwebsite=true - build_tcc: | cd muon CC=tcc tools/ci/bootstrap.sh build-tcc @@ -81,14 +81,12 @@ tasks: CC=gcc build/muon setup \ -Dbuildtype=minsize \ -Dstatic=true \ - -Dsamurai=enabled \ -Dlibcurl=disabled \ -Dlibarchive=disabled \ build-small samu -C build-small - release: | cd muon - tools/ci/git_archive_with_samurai.sh muon-edge+samurai tools/ci/prepare_release_docs.sh build tools/ci/prepare_binary.sh build edge-amd64-linux-static tools/ci/prepare_binary.sh build-small edge-amd64-linux-static-small @@ -98,12 +96,10 @@ tasks: build/muon-edge-amd64-linux-static \ build/muon-edge-amd64-linux-static.md5 \ build-small/muon-edge-amd64-linux-static-small \ - build-small/muon-edge-amd64-linux-static-small.md5 \ - muon-edge+samurai.tar.gz + build-small/muon-edge-amd64-linux-static-small.md5 artifacts: - muon/build/muon-edge-amd64-linux-static - muon/build/muon-edge-amd64-linux-static.md5 - muon/build-small/muon-edge-amd64-linux-static-small - muon/build-small/muon-edge-amd64-linux-static-small.md5 - muon/build/doc/docs/man.tar.gz - - muon/muon-edge+samurai.tar.gz diff --git a/.builds/netbsd.yml b/.builds/netbsd.yml index 4e28f9e9c..bdc103660 100644 --- a/.builds/netbsd.yml +++ b/.builds/netbsd.yml @@ -3,10 +3,6 @@ image: netbsd/latest packages: - - curl - - libarchive - - pkgconf - - samurai sources: - https://git.sr.ht/~lattis/muon tasks: @@ -14,7 +10,7 @@ tasks: cd muon # TODO remove this when we implement rpaths export LD_LIBRARY_PATH=/usr/pkg/lib - tools/ci/fullbootstrap.sh + tools/ci/bootstrap.sh build - test: | cd muon/build # TODO remove this when we implement rpaths diff --git a/README.md b/README.md index 3c6d41855..3158f415f 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,6 @@ Patches and bug reports welcome! Essential: - `c99` -- a ninja-compatible build tool (`samu` can be optionally bootstrapped with - `tools/bootstrap_ninja.sh`) For `pkgconf` support: @@ -90,19 +88,14 @@ Stage 1: ./bootstrap.sh build ``` -Optionally, if your system does not provide a ninja-compatible build tool, you -may use the provided ninja bootstrapping script. - -``` -./tools/bootstrap_ninja.sh build -ninja=build/samu -``` +This will by default build a copy of samurai into the resulting executable. To +disable this behavior use `CFLAGS=-DBOOTSTRAP_NO_SAMU`. Stage 2: ``` build/muon setup build -$ninja -C build +build/muon -C build samu build/muon -C build test build/muon -C build install ``` diff --git a/meson_options.txt b/meson_options.txt index 27e9afeb1..5034ac0a1 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -22,7 +22,7 @@ option( option( 'samurai', type: 'feature', - value: 'disabled', + value: 'enabled', description: 'embed samurai into the muon executable', ) option( diff --git a/src/amalgam.c b/src/amalgam.c index ec7cb5245..f4fd68a78 100644 --- a/src/amalgam.c +++ b/src/amalgam.c @@ -9,8 +9,6 @@ #define __EXTENSIONS__ #endif -#define NO_GETLOADAVG - #include "args.c" #include "backend/backend.c" #include "backend/common_args.c" @@ -32,19 +30,6 @@ #include "external/bestline_null.c" #include "external/libarchive_null.c" #include "external/libcurl_null.c" -#include "external/samurai.c" -#include "external/samurai/build.c" -#include "external/samurai/deps.c" -#include "external/samurai/env.c" -#include "external/samurai/graph.c" -#include "external/samurai/htab.c" -#include "external/samurai/log.c" -#include "external/samurai/parse.c" -#include "external/samurai/samu.c" -#include "external/samurai/scan.c" -#include "external/samurai/tool.c" -#include "external/samurai/tree.c" -#include "external/samurai/util.c" #include "external/tinyjson_null.c" #include "formats/editorconfig.c" #include "formats/ini.c" @@ -148,3 +133,22 @@ #else #include "external/libpkgconf_null.c" #endif + +#ifndef BOOTSTRAP_NO_SAMU +#define NO_GETLOADAVG +#include "external/samurai.c" +#include "external/samurai/build.c" +#include "external/samurai/deps.c" +#include "external/samurai/env.c" +#include "external/samurai/graph.c" +#include "external/samurai/htab.c" +#include "external/samurai/log.c" +#include "external/samurai/parse.c" +#include "external/samurai/samu.c" +#include "external/samurai/scan.c" +#include "external/samurai/tool.c" +#include "external/samurai/tree.c" +#include "external/samurai/util.c" +#else +#include "external/samurai_null.c" +#endif diff --git a/subprojects/samurai.wrap b/subprojects/samurai.wrap deleted file mode 100644 index f08f9bbe1..000000000 --- a/subprojects/samurai.wrap +++ /dev/null @@ -1,7 +0,0 @@ -; SPDX-FileCopyrightText: Stone Tickle -; SPDX-License-Identifier: GPL-3.0-only -; -[wrap-file] -source_filename = samurai-1.2-32-g81cef5d.tar.gz -source_url = https://mochiro.moe/wrap/samurai-1.2-32-g81cef5d.tar.gz -source_hash = 68f300b5272ac1e1a58ef38b693b91e81d00d0641042b55b46f8976387690a87 diff --git a/tools/bootstrap_ninja.sh b/tools/bootstrap_ninja.sh index 1ffb1b1ed..8306c0aa7 100755 --- a/tools/bootstrap_ninja.sh +++ b/tools/bootstrap_ninja.sh @@ -2,37 +2,4 @@ # SPDX-FileCopyrightText: Stone Tickle # SPDX-License-Identifier: GPL-3.0-only -# Requirements: -# - c99 -# - sh -# For automatic fetching of samurai sources: -# - curl or wget -# - tar capable of extracting a .tar.gz -# - mv - -set -eux - -dir="$1" -mkdir -p "$dir" - -# Keep in sync with subprojects/samurai.wrap -source_filename="samurai-1.2-32-g81cef5d.tar.gz" -source_url="https://mochiro.moe/wrap/samurai-1.2-32-g81cef5d.tar.gz" - -if [ ! -d subprojects/samurai ]; then - if command -v curl >/dev/null; then - curl -o "$dir/$source_filename" ${CURLOPTS:-} "$source_url" - elif command -v wget >/dev/null; then - wget -O "$dir/$source_filename" ${WGETOPTS:-} "$source_url" - else - set +x - echo "Failed to automatically fetch samurai sources." - echo "Please download and extract $source_url to subprojects/samurai." - exit 1 - fi - - tar xvf "$dir/$source_filename" - mv "samurai" "subprojects" -fi - -${CC:-c99} ${CFLAGS:-} ${LDFLAGS:-} -Isubprojects/samurai subprojects/samurai/*.c -o "$dir/samu" +echo "This tool is no longer necessary, muon comes with a bundled samu by default." diff --git a/tools/ci/bootstrap.sh b/tools/ci/bootstrap.sh index 4bc6b4e79..9b8590629 100755 --- a/tools/ci/bootstrap.sh +++ b/tools/ci/bootstrap.sh @@ -4,9 +4,10 @@ set -eux -# initial build -./bootstrap.sh "$1" +build="$1" +shift -# get curl and zlib -build/muon setup "$1" -samu -C "$1" +./bootstrap.sh "$build" + +build/muon setup "$@" "$build" +build/muon -C "$build" samu diff --git a/tools/ci/fullbootstrap.sh b/tools/ci/fullbootstrap.sh deleted file mode 100755 index 6d02f5650..000000000 --- a/tools/ci/fullbootstrap.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: Stone Tickle -# SPDX-License-Identifier: GPL-3.0-only - -set -eux - -tools/ci/bootstrap.sh build - -# enable samurai wrap -build/muon setup -Dsamurai=enabled "$@" build -samu -C build diff --git a/tools/ci/git_archive_with_samurai.sh b/tools/ci/git_archive_with_samurai.sh deleted file mode 100755 index c55811580..000000000 --- a/tools/ci/git_archive_with_samurai.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: Stone Tickle -# SPDX-License-Identifier: GPL-3.0-only - -archive="$1" - -git archive --format=tar --prefix="$archive/" HEAD > "${archive}.tar" -find subprojects/samurai -type f | tar -u -T - \ - --transform="s|^|$archive/|g" \ - -f "${archive}.tar" \ - --owner=0 \ - --group=0 - -gzip -f "${archive}.tar" diff --git a/tools/ci/solaris11.sh b/tools/ci/solaris11.sh index c546e4d3f..af1147ecf 100755 --- a/tools/ci/solaris11.sh +++ b/tools/ci/solaris11.sh @@ -24,10 +24,9 @@ build() { export CC=gcc export CFLAGS="-D_POSIX_C_SOURCE=200112L -D__EXTENSIONS__" - CURLOPTS=-k tools/bootstrap_ninja.sh build ./bootstrap.sh build - build/muon setup -Dbestline=disabled -Dsamurai=enabled build - build/samu -C build + build/muon setup build + build/muon -C build samu build/muon -C build test -d dots -s lang -j$(nproc) } From 232b5b83831d67187a1928d7dfa3dde6e5cb5eba Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Mon, 8 Jan 2024 10:21:12 -0500 Subject: [PATCH 06/52] port samu build to use run_cmd_ctx --- src/external/samurai/build.c | 176 ++++++++++------------------------- 1 file changed, 49 insertions(+), 127 deletions(-) diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index 371e53e51..62cf44d74 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -24,6 +24,8 @@ #include #include "external/samurai/ctx.h" +#include "log.h" +#include "platform/run_cmd.h" #include "external/samurai/build.h" #include "external/samurai/deps.h" @@ -35,11 +37,9 @@ struct samu_job { struct samu_string *cmd; struct samu_edge *edge; - struct samu_buffer buf; size_t next; - pid_t pid; - int fd; - bool failed; + struct run_cmd_ctx cmd_ctx; + bool failed, running; }; void @@ -272,15 +272,12 @@ samu_printstatus(struct samu_ctx *ctx, struct samu_edge *e, struct samu_string * samu_puts(ctx, description->s); } -static int +static bool samu_jobstart(struct samu_ctx *ctx, struct samu_job *j, struct samu_edge *e) { - extern char **environ; size_t i; struct samu_node *n; struct samu_string *rspfile, *content; - int fd[2]; - posix_spawn_file_actions_t actions; char *argv[] = {"/bin/sh", "-c", NULL, NULL}; ++ctx->build.nstarted; @@ -288,76 +285,43 @@ samu_jobstart(struct samu_ctx *ctx, struct samu_job *j, struct samu_edge *e) n = e->out[i]; if (n->mtime == SAMU_MTIME_MISSING) { if (samu_makedirs(n->path, true) < 0) - goto err0; + return false; } } + rspfile = samu_edgevar(ctx, e, "rspfile", false); if (rspfile) { content = samu_edgevar(ctx, e, "rspfile_content", true); if (samu_writefile(rspfile->s, content) < 0) - goto err0; + return false; } - if (pipe(fd) < 0) { - samu_warn("pipe:"); - goto err1; - } j->edge = e; j->cmd = samu_edgevar(ctx, e, "command", true); - j->fd = fd[0]; + j->cmd_ctx = (struct run_cmd_ctx) { + .flags = run_cmd_ctx_flag_async, + }; + + if (e->pool == &ctx->consolepool) { + j->cmd_ctx.flags |= run_cmd_ctx_flag_dont_capture; + } + argv[2] = j->cmd->s; if (!ctx->build.consoleused) samu_printstatus(ctx, e, j->cmd); - if ((errno = posix_spawn_file_actions_init(&actions))) { - samu_warn("posix_spawn_file_actions_init:"); - goto err2; - } - if ((errno = posix_spawn_file_actions_addclose(&actions, fd[0]))) { - samu_warn("posix_spawn_file_actions_addclose:"); - goto err3; - } - if (e->pool != &ctx->consolepool) { - if ((errno = posix_spawn_file_actions_addopen(&actions, 0, "/dev/null", O_RDONLY, 0))) { - samu_warn("posix_spawn_file_actions_addopen:"); - goto err3; - } - if ((errno = posix_spawn_file_actions_adddup2(&actions, fd[1], 1))) { - samu_warn("posix_spawn_file_actions_adddup2:"); - goto err3; - } - if ((errno = posix_spawn_file_actions_adddup2(&actions, fd[1], 2))) { - samu_warn("posix_spawn_file_actions_adddup2:"); - goto err3; - } - if ((errno = posix_spawn_file_actions_addclose(&actions, fd[1]))) { - samu_warn("posix_spawn_file_actions_addclose:"); - goto err3; - } - } - if ((errno = posix_spawn(&j->pid, argv[0], &actions, NULL, argv, environ))) { - samu_warn("posix_spawn %s:", j->cmd->s); - goto err3; + if (!run_cmd_argv(&j->cmd_ctx, argv, 0, 0)) { + samu_warn("failed to start job: %s", j->cmd_ctx.err_msg); + j->failed = true; + return false; } - posix_spawn_file_actions_destroy(&actions); - close(fd[1]); + j->failed = false; if (e->pool == &ctx->consolepool) ctx->build.consoleused = true; - return j->fd; - -err3: - posix_spawn_file_actions_destroy(&actions); -err2: - close(fd[0]); - close(fd[1]); -err1: - if (rspfile && !ctx->buildopts.keeprsp) - remove(rspfile->s); -err0: - return -1; + return true; } static void @@ -438,31 +402,19 @@ samu_edgedone(struct samu_ctx *ctx, struct samu_edge *e) static void samu_jobdone(struct samu_ctx *ctx, struct samu_job *j) { - int status; struct samu_edge *e, *new; struct samu_pool *p; ++ctx->build.nfinished; - if (waitpid(j->pid, &status, 0) < 0) { - samu_warn("waitpid %d:", j->pid); - j->failed = true; - } else if (WIFEXITED(status)) { - if (WEXITSTATUS(status) != 0) { - samu_warn("job failed: %s", j->cmd->s); - j->failed = true; + if (!ctx->build.consoleused || j->failed) { + if (j->cmd_ctx.out.len) { + fwrite(j->cmd_ctx.out.buf, 1, j->cmd_ctx.out.len, stdout); + } + + if (j->cmd_ctx.err.len) { + fwrite(j->cmd_ctx.err.buf, 1, j->cmd_ctx.err.len, stdout); } - } else if (WIFSIGNALED(status)) { - samu_warn("job terminated due to signal %d: %s", WTERMSIG(status), j->cmd->s); - j->failed = true; - } else { - /* cannot happen according to POSIX */ - samu_warn("job status unknown: %s", j->cmd->s); - j->failed = true; } - close(j->fd); - if (j->buf.len && (!ctx->build.consoleused || j->failed)) - fwrite(j->buf.data, 1, j->buf.len, stdout); - j->buf.len = 0; e = j->edge; if (e->pool) { p = e->pool; @@ -483,42 +435,6 @@ samu_jobdone(struct samu_ctx *ctx, struct samu_job *j) samu_edgedone(ctx, e); } -/* returns whether a job still has work to do. if not, sets j->failed */ -static bool -samu_jobwork(struct samu_ctx *ctx, struct samu_job *j) -{ - char *newdata; - size_t newcap; - ssize_t n; - - if (j->buf.cap - j->buf.len < BUFSIZ / 2) { - newcap = j->buf.cap + BUFSIZ; - newdata = samu_arena_realloc(&ctx->arena, j->buf.data, j->buf.cap, newcap); - if (!newdata) { - samu_warn("realloc:"); - goto kill; - } - j->buf.cap = newcap; - j->buf.data = newdata; - } - n = read(j->fd, j->buf.data + j->buf.len, j->buf.cap - j->buf.len); - if (n > 0) { - j->buf.len += n; - return true; - } - if (n == 0) - goto done; - samu_warn("read:"); - -kill: - kill(j->pid, SIGTERM); - j->failed = true; -done: - samu_jobdone(ctx, j); - - return false; -} - /* queries the system load average */ static double samu_queryload(void) @@ -541,7 +457,6 @@ void samu_build(struct samu_ctx *ctx) { struct samu_job *jobs = NULL; - struct pollfd *fds = NULL; size_t i, next = 0, jobslen = 0, maxjobs = ctx->buildopts.maxjobs, numjobs = 0, numfail = 0; struct samu_edge *e; @@ -553,7 +468,7 @@ samu_build(struct samu_ctx *ctx) samu_formatstatus(ctx, NULL, 0); ctx->build.nstarted = 0; - for (;;) { + while (true) { /* limit number of of jobs based on load */ if (ctx->buildopts.maxload) maxjobs = samu_queryload() > ctx->buildopts.maxload ? 1 : ctx->buildopts.maxjobs; @@ -577,36 +492,43 @@ samu_build(struct samu_ctx *ctx) if (newjobslen > ctx->buildopts.maxjobs) newjobslen = ctx->buildopts.maxjobs; jobs = samu_xreallocarray(&ctx->arena, jobs, jobslen, newjobslen, sizeof(jobs[0])); - fds = samu_xreallocarray(&ctx->arena, fds, jobslen, newjobslen, sizeof(fds[0])); jobslen = newjobslen; for (i = next; i < jobslen; ++i) { - jobs[i].buf.data = NULL; - jobs[i].buf.len = 0; - jobs[i].buf.cap = 0; jobs[i].next = i + 1; - fds[i].fd = -1; - fds[i].events = POLLIN; } } - fds[next].fd = samu_jobstart(ctx, &jobs[next], e); - if (fds[next].fd < 0) { + + if (!samu_jobstart(ctx, &jobs[next], e)) { samu_warn("job failed to start"); ++numfail; } else { + jobs[next].running = true; next = jobs[next].next; ++numjobs; } } if (numjobs == 0) break; - if (poll(fds, jobslen, 5000) < 0) - samu_fatal("poll:"); for (i = 0; i < jobslen; ++i) { - if (!fds[i].revents || samu_jobwork(ctx, &jobs[i])) + if (!jobs[i].running) { + continue; + } + + enum run_cmd_state state = run_cmd_collect(&jobs[i].cmd_ctx); + if (state == run_cmd_running) { continue; + } + + jobs[i].running = false; + if (state == run_cmd_error) { + jobs[i].failed = true; + } + + samu_jobdone(ctx, &jobs[i]); + run_cmd_ctx_destroy(&jobs[i].cmd_ctx); + --numjobs; jobs[i].next = next; - fds[i].fd = -1; next = i; if (jobs[i].failed) ++numfail; From 3b563024fbb2e3bc0d688df1c13085096c7da0d7 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Mon, 8 Jan 2024 10:21:28 -0500 Subject: [PATCH 07/52] run_cmd posix: check if child terminated by signal --- src/platform/posix/run_cmd.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/platform/posix/run_cmd.c b/src/platform/posix/run_cmd.c index 91620e808..377beeefd 100644 --- a/src/platform/posix/run_cmd.c +++ b/src/platform/posix/run_cmd.c @@ -162,8 +162,14 @@ run_cmd_collect(struct run_cmd_ctx *ctx) if (WIFEXITED(status)) { ctx->status = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + // TODO: it may be helpful to communicate the signal that + // caused the command to terminate, this is available with + // `WTERMSIG(status)` + ctx->err_msg = "command terminated due to signal"; + return run_cmd_error; } else { - ctx->err_msg = "child exited abnormally"; + ctx->err_msg = "command exited abnormally"; return run_cmd_error; } From 4489187a825135a32088ae9f3fe8d78605dcdd98 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:28:13 -0500 Subject: [PATCH 08/52] remove some more cleanup functions that do nothing --- include/external/samurai/htab.h | 1 - include/external/samurai/util.h | 3 --- src/external/samurai/env.c | 1 - src/external/samurai/graph.c | 7 ------- src/external/samurai/htab.c | 15 --------------- src/external/samurai/util.c | 5 ----- 6 files changed, 32 deletions(-) diff --git a/include/external/samurai/htab.h b/include/external/samurai/htab.h index a02fe4679..2af25b1b6 100644 --- a/include/external/samurai/htab.h +++ b/include/external/samurai/htab.h @@ -12,7 +12,6 @@ void samu_htabkey(struct samu_hashtablekey *, const char *, size_t); struct samu_hashtable *samu_mkhtab(struct samu_arena *a, size_t cap); -void samu_delhtab(struct samu_hashtable *, void(void *)); void **samu_htabput(struct samu_arena *a, struct samu_hashtable *h, struct samu_hashtablekey *k); void *samu_htabget(struct samu_hashtable *, struct samu_hashtablekey *); diff --git a/include/external/samurai/util.h b/include/external/samurai/util.h index e2afb8a3b..f7bb5e86c 100644 --- a/include/external/samurai/util.h +++ b/include/external/samurai/util.h @@ -35,9 +35,6 @@ void samu_bufadd(struct samu_arena *a, struct samu_buffer *buf, char c); * s, but not initialized. */ struct samu_string *samu_mkstr(struct samu_arena *a, size_t n); -/* delete an unevaluated string */ -void samu_delevalstr(void *); - /* canonicalizes the given path by removing duplicate slashes, and * folding '/.' and 'foo/..' */ void samu_canonpath(struct samu_string *); diff --git a/src/external/samurai/env.c b/src/external/samurai/env.c index 712e5f393..e44d8787a 100644 --- a/src/external/samurai/env.c +++ b/src/external/samurai/env.c @@ -120,7 +120,6 @@ samu_enveval(struct samu_ctx *ctx, struct samu_environment *env, struct samu_eva n += p->str->n; } res = samu_merge(ctx, str, n); - samu_delevalstr(str); return res; } diff --git a/src/external/samurai/graph.c b/src/external/samurai/graph.c index 4e3618b47..a8a718a15 100644 --- a/src/external/samurai/graph.c +++ b/src/external/samurai/graph.c @@ -20,18 +20,11 @@ #include "external/samurai/htab.h" #include "external/samurai/util.h" -static void -samu_delnode(void *p) -{ -} - void samu_graphinit(struct samu_ctx *ctx) { struct samu_edge *e; - /* delete old nodes and edges in case we rebuilt the manifest */ - samu_delhtab(ctx->graph.allnodes, samu_delnode); while (ctx->graph.alledges) { e = ctx->graph.alledges; ctx->graph.alledges = e->allnext; diff --git a/src/external/samurai/htab.c b/src/external/samurai/htab.c index dad28d2e7..5d25a96fd 100644 --- a/src/external/samurai/htab.c +++ b/src/external/samurai/htab.c @@ -49,21 +49,6 @@ samu_mkhtab(struct samu_arena *a, size_t cap) return h; } -void -samu_delhtab(struct samu_hashtable *h, void del(void *)) -{ - size_t i; - - if (!h) - return; - if (del) { - for (i = 0; i < h->cap; ++i) { - if (h->keys[i].str) - del(h->vals[i]); - } - } -} - static bool samu_keyequal(struct samu_hashtablekey *k1, struct samu_hashtablekey *k2) { diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index ddcb32e6c..03d28faa8 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -229,11 +229,6 @@ samu_mkstr(struct samu_arena *a, size_t n) return str; } -void -samu_delevalstr(void *ptr) -{ -} - void samu_canonpath(struct samu_string *path) { From 4eec0d7cd956c53ad4202856850316d6ce05e4a5 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:40:14 -0500 Subject: [PATCH 09/52] move mtime calculation to platform/ --- include/platform/filesystem.h | 2 ++ src/external/samurai/graph.c | 31 +++++++++---------------------- src/platform/posix/filesystem.c | 28 ++++++++++++++++++++++++++++ src/platform/windows/filesystem.c | 14 ++++++++++++++ 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/include/platform/filesystem.h b/include/platform/filesystem.h index 7058cbc9a..034e5a410 100644 --- a/include/platform/filesystem.h +++ b/include/platform/filesystem.h @@ -33,6 +33,8 @@ struct workspace; struct sbuf; bool fs_stat(const char *path, struct stat *sb); +enum fs_mtime_result { fs_mtime_result_ok, fs_mtime_result_not_found, fs_mtime_result_err }; +enum fs_mtime_result fs_mtime(const char *path, int64_t *mtime); bool fs_exists(const char *path); bool fs_file_exists(const char *path); bool fs_symlink_exists(const char *path); diff --git a/src/external/samurai/graph.c b/src/external/samurai/graph.c index a8a718a15..bb539e4ee 100644 --- a/src/external/samurai/graph.c +++ b/src/external/samurai/graph.c @@ -7,13 +7,10 @@ #include "compat.h" #include -#include -#include -#include #include -#include #include "external/samurai/ctx.h" +#include "platform/filesystem.h" #include "external/samurai/env.h" #include "external/samurai/graph.h" @@ -73,25 +70,15 @@ samu_nodeget(struct samu_ctx *ctx, const char *path, size_t len) void samu_nodestat(struct samu_node *n) { - struct stat st; - - if (stat(n->path->s, &st) < 0) { - if (errno != ENOENT) - samu_fatal("stat %s:", n->path->s); + switch (fs_mtime(n->path->s, &n->mtime)) { + case fs_mtime_result_ok: + return; + case fs_mtime_result_not_found: n->mtime = SAMU_MTIME_MISSING; - } else { -#ifdef __APPLE__ - n->mtime = (int64_t)st.st_mtime * 1000000000 + st.st_mtimensec; -/* -Illumos hides the members of st_mtim when you define _POSIX_C_SOURCE -since it has not been updated to support POSIX.1-2008: -https://www.illumos.org/issues/13327 -*/ -#elif defined(__sun) && !defined(__EXTENSIONS__) - n->mtime = (int64_t)st.st_mtim.__tv_sec * 1000000000 + st.st_mtim.__tv_nsec; -#else - n->mtime = (int64_t)st.st_mtim.tv_sec * 1000000000 + st.st_mtim.tv_nsec; -#endif + return; + case fs_mtime_result_err: + samu_fatal("stat %s:", n->path->s); + return; } } diff --git a/src/platform/posix/filesystem.c b/src/platform/posix/filesystem.c index c52f06917..b80263c45 100644 --- a/src/platform/posix/filesystem.c +++ b/src/platform/posix/filesystem.c @@ -31,6 +31,34 @@ fs_lstat(const char *path, struct stat *sb) return true; } +enum fs_mtime_result +fs_mtime(const char *path, int64_t *mtime) +{ + struct stat st; + + if (stat(path, &st) < 0) { + if (errno != ENOENT) { + LOG_E("failed stat(%s): %s", path, strerror(errno)); + return fs_mtime_result_err ; + } + return fs_mtime_result_not_found; + } else { +#ifdef __APPLE__ + *mtime = (int64_t)st.st_mtime * 1000000000 + st.st_mtimensec; +/* +Illumos hides the members of st_mtim when you define _POSIX_C_SOURCE +since it has not been updated to support POSIX.1-2008: +https://www.illumos.org/issues/13327 +*/ +#elif defined(__sun) && !defined(__EXTENSIONS__) + *mtime = (int64_t)st.st_mtim.__tv_sec * 1000000000 + st.st_mtim.__tv_nsec; +#else + *mtime = (int64_t)st.st_mtim.tv_sec * 1000000000 + st.st_mtim.tv_nsec; +#endif + return fs_mtime_result_ok; + } +} + bool fs_exists(const char *path) { diff --git a/src/platform/windows/filesystem.c b/src/platform/windows/filesystem.c index f14906a2a..fad5355ee 100644 --- a/src/platform/windows/filesystem.c +++ b/src/platform/windows/filesystem.c @@ -494,3 +494,17 @@ fs_find_cmd(struct workspace *wk, struct sbuf *buf, const char *cmd) return false; } + +enum fs_mtime_result +fs_mtime(const char *path, int64_t *mtime) +{ + WIN32_FILE_ATTRIBUTE_DATA d; + ULARGE_INTEGER t; + if (!GetFileAttributesEx(name, GetFileExInfoStandard, &d)) { + return fs_mtime_result_not_found; + } + + t.LowPart = d.ftLastWriteTime.dwLowDateTime; + t.HighPart = d.ftLastWriteTime.dwHighDateTime; + return t.QuadPart * 100; +} From a43f5c95c32138a9ca08e6b3c31285aa3a66683d Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:46:15 -0500 Subject: [PATCH 10/52] change scan.c to use a buffer directly --- include/external/samurai/ctx.h | 4 +++- include/external/samurai/scan.h | 3 ++- src/external/samurai/scan.c | 35 +++++++++++++++++++-------------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/include/external/samurai/ctx.h b/include/external/samurai/ctx.h index 5192f8074..21110a706 100644 --- a/include/external/samurai/ctx.h +++ b/include/external/samurai/ctx.h @@ -11,6 +11,7 @@ #include #include +#include "platform/filesystem.h" #include "platform/timer.h" struct samu_buffer { @@ -58,9 +59,10 @@ enum samu_token { }; struct samu_scanner { - FILE *f; + struct source src; const char *path; int chr, line, col; + uint32_t src_i; }; struct samu_node { diff --git a/include/external/samurai/scan.h b/include/external/samurai/scan.h index 5da3e7f5e..198b557af 100644 --- a/include/external/samurai/scan.h +++ b/include/external/samurai/scan.h @@ -10,7 +10,8 @@ void samu_scaninit(struct samu_scanner *, const char *); void samu_scanclose(struct samu_scanner *); -void samu_scanerror(struct samu_scanner *, const char *, ...); +void samu_scanerror(struct samu_scanner *, const char *, ...) +MUON_ATTR_FORMAT(printf, 2, 3); int samu_scankeyword(struct samu_ctx *ctx, struct samu_scanner *s, char **var); char *samu_scanname(struct samu_ctx *ctx, struct samu_scanner *s); struct samu_evalstring *samu_scanstring(struct samu_ctx *ctx, struct samu_scanner *s, bool path); diff --git a/src/external/samurai/scan.c b/src/external/samurai/scan.c index 28e94388e..97c114c70 100644 --- a/src/external/samurai/scan.c +++ b/src/external/samurai/scan.c @@ -8,8 +8,6 @@ #include #include -#include -#include #include #include @@ -19,25 +17,27 @@ #include "external/samurai/scan.h" #include "external/samurai/util.h" -#undef getc -#define getc getc_unlocked - void samu_scaninit(struct samu_scanner *s, const char *path) { - s->path = path; - s->line = 1; - s->col = 1; - s->f = fopen(path, "r"); - if (!s->f) - samu_fatal("open %s:", path); - s->chr = getc(s->f); + *s = (struct samu_scanner) { + .path = path, + .line = 1, + .col = 1, + .src_i = 1, + }; + + if (!fs_read_entire_file(path, &s->src)) { + samu_fatal("failed to read %s", path); + } + + s->chr = s->src.src[0]; } void samu_scanclose(struct samu_scanner *s) { - fclose(s->f); + fs_source_destroy(&s->src); } void @@ -62,7 +62,12 @@ samu_next(struct samu_scanner *s) } else { ++s->col; } - s->chr = getc(s->f); + if (s->src_i < s->src.len) { + s->chr = s->src.src[s->src_i]; + ++s->src_i; + } else { + s->chr = EOF; + } return s->chr; } @@ -102,7 +107,7 @@ samu_singlespace(struct samu_scanner *s) samu_next(s); if (samu_newline(s)) return true; - ungetc(s->chr, s->f); + --s->src_i; s->chr = '$'; return false; case ' ': From f8054b4a27fb0b931b6547d90cc0c7baefc1bb53 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:46:41 -0500 Subject: [PATCH 11/52] move os_ functions that only path should call --- include/platform/os.h | 3 --- src/platform/path.c | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/include/platform/os.h b/include/platform/os.h index 65c306b93..b72d324a4 100644 --- a/include/platform/os.h +++ b/include/platform/os.h @@ -25,8 +25,5 @@ extern int opterr, optind, optopt; #include #endif -bool os_chdir(const char *path); -char *os_getcwd(char *buf, size_t size); int os_getopt(int argc, char * const argv[], const char *optstring); - #endif diff --git a/src/platform/path.c b/src/platform/path.c index 9596413ad..b397cfa39 100644 --- a/src/platform/path.c +++ b/src/platform/path.c @@ -23,6 +23,13 @@ static struct { struct sbuf cwd, tmp1, tmp2; } path_ctx; +// These functions are defined in platform//os.c +// +// They should only be called indirectly through path_chdir and path_cwd though +// so the prototypes were removed from os.h. +bool os_chdir(const char *path); +char *os_getcwd(char *buf, size_t size); + static void path_getcwd(void) { From 40d4ca8d98e67901142ee7e6ace62c81d57f78a9 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:47:01 -0500 Subject: [PATCH 12/52] fix arrangement of tracy calls wrt freed pointers --- src/platform/mem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/mem.c b/src/platform/mem.c index 0d9483cc5..895a10ad0 100644 --- a/src/platform/mem.c +++ b/src/platform/mem.c @@ -62,13 +62,13 @@ z_realloc(void *ptr, size_t size) { assert(size); void *ret; + TracyCFree(ptr); ret = realloc(ptr, size); if (!ret) { error_unrecoverable("realloc failed: %s", strerror(errno)); } - TracyCFree(ptr); TracyCAlloc(ret, size); PlotRSS; return ret; @@ -78,7 +78,7 @@ void z_free(void *ptr) { assert(ptr); - free(ptr); TracyCFree(ptr); + free(ptr); PlotRSS; } From a2226cee23d902ad8a0f2c928b5cfdde799026c5 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:48:38 -0500 Subject: [PATCH 13/52] use fs_write in samu_write_file --- src/external/samurai/util.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index 03d28faa8..503e8cd5d 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -6,16 +6,16 @@ #include "compat.h" +#include #include #include #include -#include #include #include +#include #include #include -#include "assert.h" #include "buf_size.h" #include "external/samurai/ctx.h" #include "log.h" @@ -325,20 +325,9 @@ samu_makedirs(struct samu_string *path, bool parent) int samu_writefile(const char *name, struct samu_string *s) { - FILE *f; - int ret; - - f = fopen(name, "w"); - if (!f) { - samu_warn("open %s:", name); + if (!fs_write(name, (uint8_t *)s->s, s->n)) { return -1; } - ret = 0; - if (s && (fwrite(s->s, 1, s->n, f) != s->n || fflush(f) != 0)) { - samu_warn("write %s:", name); - ret = -1; - } - fclose(f); - return ret; + return 0; } From 5ea94d67cfe53cd322f438c6333513882901d060 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:54:00 -0500 Subject: [PATCH 14/52] fix argument type --- src/external/samurai/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index 503e8cd5d..bf582d00c 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -130,7 +130,7 @@ samu_arena_destroy(struct samu_arena *a) } void * -samu_arena_alloc(struct samu_arena *a, uint64_t size) +samu_arena_alloc(struct samu_arena *a, size_t size) { uint64_t align = -a->i & 7; a->i += align; From 253b287aab031c98a0e937588f891f7e2d040cdb Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 10:55:18 -0500 Subject: [PATCH 15/52] use fs_mkdir functions in samu_makedirs --- src/external/samurai/util.c | 44 ++++++++++--------------------------- src/platform/filesystem.c | 2 +- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index bf582d00c..5195089d8 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -14,13 +14,14 @@ #include #include #include -#include #include "buf_size.h" +#include "error.h" #include "external/samurai/ctx.h" +#include "lang/string.h" #include "log.h" -#include "error.h" #include "platform/mem.h" +#include "platform/path.h" #include "external/samurai/util.h" @@ -288,38 +289,17 @@ samu_canonpath(struct samu_string *path) int samu_makedirs(struct samu_string *path, bool parent) { - int ret; - struct stat st; - char *s, *end; - - ret = 0; - end = path->s + path->n; - for (s = end - parent; s > path->s; --s) { - if (*s != '/' && *s) - continue; - *s = '\0'; - if (stat(path->s, &st) == 0) - break; - if (errno != ENOENT) { - samu_warn("stat %s:", path->s); - ret = -1; - break; - } - } - if (s > path->s && s < end) - *s = '/'; - while (++s <= end - parent) { - if (*s != '\0') - continue; - if (ret == 0 && mkdir(path->s, 0777) < 0 && errno != EEXIST) { - samu_warn("mkdir %s:", path->s); - ret = -1; - } - if (s < end) - *s = '/'; + bool ok; + if (parent) { + SBUF_manual(dirname); + path_dirname(0, &dirname, path->s); + ok = fs_mkdir_p(dirname.buf); + sbuf_destroy(&dirname); + } else { + ok = fs_mkdir(path->s); } - return ret; + return ok ? 0 : -1; } int diff --git a/src/platform/filesystem.c b/src/platform/filesystem.c index b7b5f165d..80fb58680 100644 --- a/src/platform/filesystem.c +++ b/src/platform/filesystem.c @@ -52,7 +52,7 @@ fs_mkdir_p(const char *path) SBUF_manual(buf); path_copy(NULL, &buf, path); - assert(len > 1); + assert(len >= 1); for (i = 1; i < len; ++i) { if (buf.buf[i] == PATH_SEP) { From 5684611e58fda5b85c3bb4b5edcb61f97c4ba6d5 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 11:11:31 -0500 Subject: [PATCH 16/52] use path_chdir rather than chdir --- src/external/samurai/samu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c index 5d86d55af..c1a8ffdef 100644 --- a/src/external/samurai/samu.c +++ b/src/external/samurai/samu.c @@ -16,6 +16,7 @@ #include "assert.h" #include "buf_size.h" #include "external/samurai/ctx.h" +#include "platform/path.h" #include "external/samurai.h" #include "external/samurai/arg.h" @@ -184,7 +185,7 @@ samu_main(int argc, char *argv[], struct samu_opts *opts) case 'C': arg = SAMU_EARGF(samu_usage(ctx)); /* samu_warn("entering directory '%s'", arg); */ - if (chdir(arg) < 0) + if (!path_chdir(arg)) samu_fatal("chdir:"); break; case 'd': From aea89271b74825c955d48d07f6f0852b17835efd Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 11:11:56 -0500 Subject: [PATCH 17/52] add os_ functions for querying job spawning info --- include/platform/os.h | 7 +++++++ src/external/samurai/build.c | 21 ++------------------- src/external/samurai/samu.c | 19 ++----------------- src/platform/posix/os.c | 34 ++++++++++++++++++++++++++++++++++ src/platform/windows/os.c | 14 ++++++++++++++ 5 files changed, 59 insertions(+), 36 deletions(-) diff --git a/include/platform/os.h b/include/platform/os.h index b72d324a4..c3d621478 100644 --- a/include/platform/os.h +++ b/include/platform/os.h @@ -8,6 +8,7 @@ #define MUON_PLATFORM_OS_H #include +#include #ifdef _WIN32 #ifndef S_IRUSR @@ -26,4 +27,10 @@ extern int opterr, optind, optopt; #endif int os_getopt(int argc, char * const argv[], const char *optstring); + +// Returns the number of jobs to spawn. This number should be slightly larger +// than the number of cpus. +uint32_t os_parallel_job_count(void); + +double os_getloadavg(void); #endif diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index 62cf44d74..5f47512f3 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -25,6 +25,7 @@ #include "external/samurai/ctx.h" #include "log.h" +#include "platform/os.h" #include "platform/run_cmd.h" #include "external/samurai/build.h" @@ -435,24 +436,6 @@ samu_jobdone(struct samu_ctx *ctx, struct samu_job *j) samu_edgedone(ctx, e); } -/* queries the system load average */ -static double -samu_queryload(void) -{ -#ifdef NO_GETLOADAVG - return 0; -#else - double load; - - if (getloadavg(&load, 1) == -1) { - samu_warn("getloadavg:"); - load = 100.0; - } - - return load; -#endif -} - void samu_build(struct samu_ctx *ctx) { @@ -471,7 +454,7 @@ samu_build(struct samu_ctx *ctx) while (true) { /* limit number of of jobs based on load */ if (ctx->buildopts.maxload) - maxjobs = samu_queryload() > ctx->buildopts.maxload ? 1 : ctx->buildopts.maxjobs; + maxjobs = os_getloadavg() > ctx->buildopts.maxload ? 1 : ctx->buildopts.maxjobs; /* start ready edges */ while (ctx->build.work && numjobs < maxjobs && numfail < ctx->buildopts.maxfail) { e = ctx->build.work; diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c index c1a8ffdef..a121f31d2 100644 --- a/src/external/samurai/samu.c +++ b/src/external/samurai/samu.c @@ -11,11 +11,11 @@ #include #include #include -#include /* for chdir */ #include "assert.h" #include "buf_size.h" #include "external/samurai/ctx.h" +#include "platform/os.h" #include "platform/path.h" #include "external/samurai.h" @@ -223,22 +223,7 @@ samu_main(int argc, char *argv[], struct samu_opts *opts) } SAMU_ARGEND argdone: if (!ctx->buildopts.maxjobs) { -#ifdef _SC_NPROCESSORS_ONLN - int n = sysconf(_SC_NPROCESSORS_ONLN); - switch (n) { - case -1: case 0: case 1: - ctx->buildopts.maxjobs = 2; - break; - case 2: - ctx->buildopts.maxjobs = 3; - break; - default: - ctx->buildopts.maxjobs = n + 2; - break; - } -#else - ctx->buildopts.maxjobs = 2; -#endif + ctx->buildopts.maxjobs = os_parallel_job_count(); } ctx->buildopts.statusfmt = getenv("NINJA_STATUS"); diff --git a/src/platform/posix/os.c b/src/platform/posix/os.c index f73cd8c7d..d2ea77cd8 100644 --- a/src/platform/posix/os.c +++ b/src/platform/posix/os.c @@ -24,3 +24,37 @@ int os_getopt(int argc, char * const argv[], const char *optstring) { return getopt(argc, argv, optstring); } + +uint32_t +os_parallel_job_count(void) +{ +#ifdef _SC_NPROCESSORS_ONLN + int n = sysconf(_SC_NPROCESSORS_ONLN); + if (n == -1) { + return 4; + } else if (n < 2) { + return 2; + } else { + return n + 2; + } +#else + return 4; +#endif +} + +double +os_getloadavg(void) +{ +#ifdef HAVE_GETLOADAVG + double load; + + if (getloadavg(&load, 1) == -1) { + samu_warn("getloadavg:"); + load = 100.0; + } + + return load; +#else + return 0; +#endif +} diff --git a/src/platform/windows/os.c b/src/platform/windows/os.c index eb15e46f6..cccd4cbf1 100644 --- a/src/platform/windows/os.c +++ b/src/platform/windows/os.c @@ -142,3 +142,17 @@ os_getopt(int argc, char * const argv[], const char *optstring) } return c; } + +uint32_t +os_parallel_job_count(void) +{ + // TODO: this needs a real implementation + return 4; +} + +double +os_getloadavg(void) +{ + // TODO: this needs a real implementation + return 0; +} From b6381012b04aa30bf7cbb33a4ed755133046db24 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 11:13:51 -0500 Subject: [PATCH 18/52] use os_parallel_job_count() in cmd_test to set default job count --- src/cmd_test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cmd_test.c b/src/cmd_test.c index 204e170ca..e88865187 100644 --- a/src/cmd_test.c +++ b/src/cmd_test.c @@ -18,6 +18,7 @@ #include "log.h" #include "platform/filesystem.h" #include "platform/mem.h" +#include "platform/os.h" #include "platform/path.h" #include "platform/run_cmd.h" #include "platform/term.h" @@ -897,7 +898,7 @@ tests_run(struct test_options *opts, const char *argv0) wk.argv0 = argv0; if (!opts->jobs) { - opts->jobs = 4; + opts->jobs = os_parallel_job_count(); } struct run_test_ctx ctx = { From da0169078f20d5dcdb38cee605a45b26a05a8ba5 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 11:15:45 -0500 Subject: [PATCH 19/52] use path_cwd rather than getcwd --- src/external/samurai/tool.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/external/samurai/tool.c b/src/external/samurai/tool.c index b9c9e4911..75c0aff6e 100644 --- a/src/external/samurai/tool.c +++ b/src/external/samurai/tool.c @@ -16,6 +16,8 @@ #include "buf_size.h" #include "external/samurai/ctx.h" +#include "lang/string.h" +#include "platform/path.h" #include "external/samurai/arg.h" #include "external/samurai/env.h" @@ -205,7 +207,7 @@ samu_printjson(struct samu_ctx *ctx, const char *s, size_t n, bool join) static int samu_compdb(struct samu_ctx *ctx, int argc, char *argv[]) { - char dir[PATH_MAX], *p; + char *p; struct samu_edge *e; struct samu_string *cmd, *rspfile, *content; bool expandrsp = false, first = true; @@ -225,8 +227,8 @@ samu_compdb(struct samu_ctx *ctx, int argc, char *argv[]) return 2; } SAMU_ARGEND - if (!getcwd(dir, sizeof(dir))) - samu_fatal("getcwd:"); + SBUF_manual(dir); + path_cwd(0, &dir); samu_putchar(ctx, '['); for (e = ctx->graph.alledges; e; e = e->allnext) { @@ -254,7 +256,7 @@ samu_compdb(struct samu_ctx *ctx, int argc, char *argv[]) samu_putchar(ctx, ','); samu_printf(ctx, "\n {\n \"directory\": \""); - samu_printjson(ctx, dir, -1, false); + samu_printjson(ctx, dir.buf, -1, false); samu_printf(ctx, "\",\n \"command\": \""); cmd = samu_edgevar(ctx, e, "command", true); @@ -284,6 +286,7 @@ samu_compdb(struct samu_ctx *ctx, int argc, char *argv[]) if (fflush(stdout) || ferror(stdout)) samu_fatal("write failed"); + sbuf_destroy(&dir); return 0; } From 5e578587144e77c68e051df72dcbb65857499305 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 11:29:35 -0500 Subject: [PATCH 20/52] remove unused includes --- src/external/samurai/build.c | 15 --------------- src/external/samurai/deps.c | 3 --- src/external/samurai/env.c | 2 -- src/external/samurai/htab.c | 3 --- src/external/samurai/log.c | 1 - src/external/samurai/parse.c | 2 -- src/external/samurai/samu.c | 2 -- src/external/samurai/tool.c | 4 ---- src/external/samurai/tree.c | 1 - src/external/samurai/util.c | 4 ---- 10 files changed, 37 deletions(-) diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index 5f47512f3..57410048a 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -6,22 +6,7 @@ #include "compat.h" -#ifndef NO_GETLOADAVG -#define _BSD_SOURCE /* for getloadavg */ -#endif - -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include "external/samurai/ctx.h" #include "log.h" diff --git a/src/external/samurai/deps.c b/src/external/samurai/deps.c index 1207802a0..8033e4f87 100644 --- a/src/external/samurai/deps.c +++ b/src/external/samurai/deps.c @@ -9,9 +9,6 @@ #include #include #include -#include -#include -#include #include #include "external/samurai/ctx.h" diff --git a/src/external/samurai/env.c b/src/external/samurai/env.c index e44d8787a..f9162294f 100644 --- a/src/external/samurai/env.c +++ b/src/external/samurai/env.c @@ -6,8 +6,6 @@ #include "compat.h" -#include -#include #include #include "external/samurai/ctx.h" diff --git a/src/external/samurai/htab.c b/src/external/samurai/htab.c index 5d25a96fd..1e17e13fa 100644 --- a/src/external/samurai/htab.c +++ b/src/external/samurai/htab.c @@ -7,9 +7,6 @@ #include "compat.h" #include -#include -#include -#include #include #include "external/samurai/ctx.h" diff --git a/src/external/samurai/log.c b/src/external/samurai/log.c index 6d12d29aa..617e0a338 100644 --- a/src/external/samurai/log.c +++ b/src/external/samurai/log.c @@ -8,7 +8,6 @@ #include #include -#include #include #include diff --git a/src/external/samurai/parse.c b/src/external/samurai/parse.c index 75cdce41f..5972ed028 100644 --- a/src/external/samurai/parse.c +++ b/src/external/samurai/parse.c @@ -6,8 +6,6 @@ #include "compat.h" -#include -#include #include #include diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c index a121f31d2..231b704ee 100644 --- a/src/external/samurai/samu.c +++ b/src/external/samurai/samu.c @@ -7,8 +7,6 @@ #include "compat.h" #include -#include -#include #include #include diff --git a/src/external/samurai/tool.c b/src/external/samurai/tool.c index 75c0aff6e..5d2ab0756 100644 --- a/src/external/samurai/tool.c +++ b/src/external/samurai/tool.c @@ -7,12 +7,8 @@ #include "compat.h" #include -#include -#include -#include #include #include -#include #include "buf_size.h" #include "external/samurai/ctx.h" diff --git a/src/external/samurai/tree.c b/src/external/samurai/tree.c index a342a145d..d82956532 100644 --- a/src/external/samurai/tree.c +++ b/src/external/samurai/tree.c @@ -9,7 +9,6 @@ #include "compat.h" -#include #include #include "external/samurai/ctx.h" diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index 5195089d8..2bbfe40c0 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -8,10 +8,6 @@ #include #include -#include -#include -#include -#include #include #include From 40c77ef69637622146c80b8c7782802287ba7524 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 10 Jan 2024 11:39:42 -0500 Subject: [PATCH 21/52] re-add job failure message --- src/external/samurai/build.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index 57410048a..8ae2a5c7c 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -391,6 +391,10 @@ samu_jobdone(struct samu_ctx *ctx, struct samu_job *j) struct samu_edge *e, *new; struct samu_pool *p; + if (j->failed) { + samu_warn("job failed: %s", j->cmd->s); + } + ++ctx->build.nfinished; if (!ctx->build.consoleused || j->failed) { if (j->cmd_ctx.out.len) { From c13a701ba106a4cd6acad842aed6bf1080699fbd Mon Sep 17 00:00:00 2001 From: Michael Forney Date: Sun, 18 Jun 2023 00:19:30 -0700 Subject: [PATCH 22/52] Include exit status in job failure message --- src/external/samurai/build.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index 8ae2a5c7c..b6193cee7 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -392,7 +392,7 @@ samu_jobdone(struct samu_ctx *ctx, struct samu_job *j) struct samu_pool *p; if (j->failed) { - samu_warn("job failed: %s", j->cmd->s); + samu_warn("job failed with status %d: %s", j->cmd_ctx.status, j->cmd->s); } ++ctx->build.nfinished; From 7d224d0025b94ea25ef43ef1b226e0fd7e1da620 Mon Sep 17 00:00:00 2001 From: Michael Forney Date: Fri, 30 Jun 2023 14:27:59 -0700 Subject: [PATCH 23/52] deps: Handle escaped newlines in Makefile fragments --- src/external/samurai/deps.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/external/samurai/deps.c b/src/external/samurai/deps.c index 8033e4f87..06e3862cd 100644 --- a/src/external/samurai/deps.c +++ b/src/external/samurai/deps.c @@ -314,21 +314,14 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) if (++n % 2 == 0) samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); } while (c == '\\'); - switch (c) { - case '#': - /* assume no comments */ - for (; n > 2; n -= 2) - samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + if ((c == ' ' || c == '\t') && n % 2 != 0) break; - case ' ': - case '\t': - if (n % 2 != 0) - break; - /* fallthrough */ - default: - for (; n > 0; n -= 2) - samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); - continue; + for (; n > 2; n -= 2) + samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + switch (c) { + case '#': break; + case '\n': c = ' '; continue; + default: samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); continue; } break; case '$': From fdaf9a84e653f80798fd7ebdac02af5da51487f7 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 08:06:19 -0500 Subject: [PATCH 24/52] fix detection/usage of getloadavg --- src/amalgam.c | 1 - src/external/samurai/samu.c | 4 ---- src/platform/meson.build | 8 ++++++++ src/platform/posix/os.c | 12 ++++++++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/amalgam.c b/src/amalgam.c index f4fd68a78..5e1ed076e 100644 --- a/src/amalgam.c +++ b/src/amalgam.c @@ -135,7 +135,6 @@ #endif #ifndef BOOTSTRAP_NO_SAMU -#define NO_GETLOADAVG #include "external/samurai.c" #include "external/samurai/build.c" #include "external/samurai/deps.c" diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c index 231b704ee..4246836df 100644 --- a/src/external/samurai/samu.c +++ b/src/external/samurai/samu.c @@ -63,9 +63,6 @@ samu_debugflag(struct samu_ctx *ctx, const char *flag) static void samu_loadflag(struct samu_ctx *ctx, const char *flag) { -#ifdef NO_GETLOADAVG - samu_warn("job scheduling based on load average is not implemented"); -#else double value; char *end; errno = 0; @@ -74,7 +71,6 @@ samu_loadflag(struct samu_ctx *ctx, const char *flag) if (*end || value < 0 || errno != 0) samu_fatal("invalid -l parameter"); ctx->buildopts.maxload = value; -#endif } static void diff --git a/src/platform/meson.build b/src/platform/meson.build index d58414d26..27b71c0d2 100644 --- a/src/platform/meson.build +++ b/src/platform/meson.build @@ -38,4 +38,12 @@ if platform == 'posix' else platform_sources += files('posix/rpath_fixer.c') endif + + if cc.has_function( + 'getloadavg', + args: ['-D_BSD_SOURCE'], + prefix: '#include ', + ) + c_args += '-DMUON_HAVE_GETLOADAVG' + endif endif diff --git a/src/platform/posix/os.c b/src/platform/posix/os.c index d2ea77cd8..34233b37f 100644 --- a/src/platform/posix/os.c +++ b/src/platform/posix/os.c @@ -6,8 +6,16 @@ #include "compat.h" +#ifdef MUON_HAVE_GETLOADAVG +#include +#define _BSD_SOURCE +#include +#include +#endif + #include +#include "log.h" #include "platform/os.h" bool os_chdir(const char *path) @@ -45,11 +53,11 @@ os_parallel_job_count(void) double os_getloadavg(void) { -#ifdef HAVE_GETLOADAVG +#ifdef MUON_HAVE_GETLOADAVG double load; if (getloadavg(&load, 1) == -1) { - samu_warn("getloadavg:"); + LOG_W("failed: getloadavg: %s", strerror(errno)); load = 100.0; } From 8a03815c99fae2a42279d864f82b744886ef2ac2 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 08:10:05 -0500 Subject: [PATCH 25/52] fix macos build --- .github/workflows/macos.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index e99da1eed..483a94133 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -21,15 +21,15 @@ jobs: - name: bootstrap_deps run: | set -x - CFLAGS=-DNO_GETLOADAVG tools/bootstrap_ninja.sh build + mkdir -p build curl -L -o build/pkgconf-1.9.3.tar.gz https://github.com/pkgconf/pkgconf/archive/refs/tags/pkgconf-1.9.3.tar.gz tar xvf build/pkgconf-1.9.3.tar.gz mv pkgconf-pkgconf-1.9.3 subprojects/pkgconf - name: bootstrap run: | ./bootstrap.sh build - PATH="build:$PATH" CFLAGS=-DNO_GETLOADAVG build/muon setup -Dsamurai=enabled build - build/samu -C build + PATH="build:$PATH" build/muon setup build + build/muon -C build samu - name: test run: | build/muon -C build test || true From d5b7d32107b93ad559a3c1cc4a24dfdc87882266 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 13:44:03 -0500 Subject: [PATCH 26/52] fix variable name typo --- src/platform/windows/filesystem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/windows/filesystem.c b/src/platform/windows/filesystem.c index fad5355ee..18d41d678 100644 --- a/src/platform/windows/filesystem.c +++ b/src/platform/windows/filesystem.c @@ -500,7 +500,7 @@ fs_mtime(const char *path, int64_t *mtime) { WIN32_FILE_ATTRIBUTE_DATA d; ULARGE_INTEGER t; - if (!GetFileAttributesEx(name, GetFileExInfoStandard, &d)) { + if (!GetFileAttributesEx(path, GetFileExInfoStandard, &d)) { return fs_mtime_result_not_found; } From 7f6f41bc77fe4d227b0e647b64240327c84947d7 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 13:44:20 -0500 Subject: [PATCH 27/52] fix samu failed job detection --- src/external/samurai/build.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index b6193cee7..3d4f7ebdf 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -492,7 +492,7 @@ samu_build(struct samu_ctx *ctx) } jobs[i].running = false; - if (state == run_cmd_error) { + if (state == run_cmd_error || jobs[i].cmd_ctx.status != 0) { jobs[i].failed = true; } From 10e551a2f355b55ec7c34a38235afaec3e93facb Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 13:44:42 -0500 Subject: [PATCH 28/52] add -v in ci --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 483a94133..fadbb4086 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -28,7 +28,7 @@ jobs: - name: bootstrap run: | ./bootstrap.sh build - PATH="build:$PATH" build/muon setup build + PATH="build:$PATH" build/muon -v setup build build/muon -C build samu - name: test run: | From c4cc3ba7b04ba0beff16a710962abe3470f36e9b Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 13:49:41 -0500 Subject: [PATCH 29/52] make compiler checks log the source they are using --- src/functions/compiler.c | 6 ++++-- src/platform/posix/os.c | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/functions/compiler.c b/src/functions/compiler.c index 7a32933f1..70392a8ff 100644 --- a/src/functions/compiler.c +++ b/src/functions/compiler.c @@ -338,13 +338,15 @@ compiler_check(struct workspace *wk, struct compiler_check_opts *opts, opts->cache_key = make_strn(wk, (const char *)sha, 32); if (!opts->src_is_path) { + L("compiling: '%s'", src); + if (!fs_write(get_cstr(wk, source_path), (const uint8_t *)src, strlen(src))) { return false; } + } else { + L("compiling: '%s'", get_cstr(wk, source_path)); } - L("compiling: '%s'", get_cstr(wk, source_path)); - if (!run_cmd(&cmd_ctx, argstr, argc, NULL, 0)) { interp_error(wk, err_node, "error: %s", cmd_ctx.err_msg); goto ret; diff --git a/src/platform/posix/os.c b/src/platform/posix/os.c index 34233b37f..6699b9117 100644 --- a/src/platform/posix/os.c +++ b/src/platform/posix/os.c @@ -7,8 +7,8 @@ #include "compat.h" #ifdef MUON_HAVE_GETLOADAVG -#include #define _BSD_SOURCE +#include #include #include #endif From 6c8cbce200b85e19ccc4bd332cf601bd976302f8 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 16:21:34 -0500 Subject: [PATCH 30/52] fix getloadavg detection on macos --- src/platform/meson.build | 10 +++++++++- src/platform/posix/os.c | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/platform/meson.build b/src/platform/meson.build index 27b71c0d2..3431437a7 100644 --- a/src/platform/meson.build +++ b/src/platform/meson.build @@ -39,9 +39,17 @@ if platform == 'posix' platform_sources += files('posix/rpath_fixer.c') endif + # getloadavg detection + + if host_machine.system() == 'darwin' + defines = [] + else + defines = ['-D_POSIX_C_SOURCE=200809L', '-D_BSD_SOURCE'] + endif + if cc.has_function( 'getloadavg', - args: ['-D_BSD_SOURCE'], + args: defines, prefix: '#include ', ) c_args += '-DMUON_HAVE_GETLOADAVG' diff --git a/src/platform/posix/os.c b/src/platform/posix/os.c index 6699b9117..35508011d 100644 --- a/src/platform/posix/os.c +++ b/src/platform/posix/os.c @@ -6,14 +6,23 @@ #include "compat.h" +#include + #ifdef MUON_HAVE_GETLOADAVG -#define _BSD_SOURCE #include -#include #include + +#if defined(__APPLE__) +// On macOS, getloadavg is unavailable if _POSIX_C_SOURCE is defined +#undef _POSIX_C_SOURCE +#else +// Otherwise assume getloadavg is available when _BSD_SOURCE is defined +#define _BSD_SOURCE #endif -#include +#include + +#endif #include "log.h" #include "platform/os.h" From 0805dff7102127b81074700eb6ad91f8052b435f Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 19:57:54 -0500 Subject: [PATCH 31/52] implement job count calculation for windows --- src/amalgam.c | 7 ++-- src/platform/meson.build | 1 + src/platform/os.c | 24 +++++++++++ src/platform/posix/os.c | 15 ++----- src/platform/windows/os.c | 83 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 src/platform/os.c diff --git a/src/amalgam.c b/src/amalgam.c index 5e1ed076e..8ee64697c 100644 --- a/src/amalgam.c +++ b/src/amalgam.c @@ -95,6 +95,7 @@ #include "opts.c" #include "platform/filesystem.c" #include "platform/mem.c" +#include "platform/os.c" #include "platform/path.c" #include "platform/run_cmd.c" #include "platform/uname.c" @@ -134,7 +135,9 @@ #include "external/libpkgconf_null.c" #endif -#ifndef BOOTSTRAP_NO_SAMU +#ifdef BOOTSTRAP_NO_SAMU +#include "external/samurai_null.c" +#else #include "external/samurai.c" #include "external/samurai/build.c" #include "external/samurai/deps.c" @@ -148,6 +151,4 @@ #include "external/samurai/tool.c" #include "external/samurai/tree.c" #include "external/samurai/util.c" -#else -#include "external/samurai_null.c" #endif diff --git a/src/platform/meson.build b/src/platform/meson.build index 3431437a7..585ac7ebf 100644 --- a/src/platform/meson.build +++ b/src/platform/meson.build @@ -5,6 +5,7 @@ platform_sources = files( 'filesystem.c', 'mem.c', + 'os.c', 'path.c', 'run_cmd.c', 'uname.c', diff --git a/src/platform/os.c b/src/platform/os.c new file mode 100644 index 000000000..9291d461f --- /dev/null +++ b/src/platform/os.c @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: Stone Tickle + * SPDX-License-Identifier: GPL-3.0-only + */ + +#include "compat.h" + +#include "platform/os.h" + +int32_t os_ncpus(void); + +uint32_t +os_parallel_job_count(void) +{ + int32_t n = os_ncpus(); + + if (n == -1) { + return -1; + } else if (n < 2) { + return 2; + } else { + return n + 2; + } +} diff --git a/src/platform/posix/os.c b/src/platform/posix/os.c index 35508011d..6deff3260 100644 --- a/src/platform/posix/os.c +++ b/src/platform/posix/os.c @@ -42,20 +42,13 @@ int os_getopt(int argc, char * const argv[], const char *optstring) return getopt(argc, argv, optstring); } -uint32_t -os_parallel_job_count(void) +int32_t +os_ncpus(void) { #ifdef _SC_NPROCESSORS_ONLN - int n = sysconf(_SC_NPROCESSORS_ONLN); - if (n == -1) { - return 4; - } else if (n < 2) { - return 2; - } else { - return n + 2; - } + return sysconf(_SC_NPROCESSORS_ONLN); #else - return 4; + return -1; #endif } diff --git a/src/platform/windows/os.c b/src/platform/windows/os.c index cccd4cbf1..c4a538015 100644 --- a/src/platform/windows/os.c +++ b/src/platform/windows/os.c @@ -143,11 +143,86 @@ os_getopt(int argc, char * const argv[], const char *optstring) return c; } -uint32_t -os_parallel_job_count(void) +static uint32_t +count_bits(ULONG_PTR bit_mask) { - // TODO: this needs a real implementation - return 4; + DWORD lshift; + uint32_t bit_count = 0; + ULONG_PTR bit_test; + DWORD i; + + lshift = sizeof(ULONG_PTR) * 8 - 1; + bit_test = (ULONG_PTR)1 << lshift; + + for (i = 0; i <= lshift; i++) { + bit_count += ((bit_mask & bit_test) ? 1 : 0); + bit_test /= 2; + } + + return bit_count; +} + +int32_t +os_ncpus(void) +{ + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION iter; + uint32_t ncpus; + DWORD length; + DWORD byte_offset; + BOOL ret; + + buffer = NULL; + length = 0UL; + ret = GetLogicalProcessorInformation(buffer, &length); + /* + * buffer and length values make this function failing + * with error being ERROR_INSUFFICIENT_BUFFER. + * Error not being ERROR_INSUFFICIENT_BUFFER is very unlikely. + */ + if (!ret) { + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + return -1; + } + /* + * Otherwise length is the size in bytes to allocate + */ + } + + buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(length); + if (!buffer) { + return -1; + } + + ret = GetLogicalProcessorInformation(buffer, &length); + /* + * Should not fail as buffer and length have the correct values, + * but anyway, we check the returned value. + */ + if (!ret) { + free(buffer); + return -1; + } + + iter = buffer; + byte_offset = 0; + ncpus = 0; + + while (byte_offset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= length) { + switch (iter->Relationship) { + case RelationProcessorCore: + ncpus += count_bits(iter->ProcessorMask); + break; + default: + break; + } + byte_offset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + iter++; + } + + free(buffer); + + return ncpus; } double From 7305678969e9f9e8d1eef0fe3c78a87b19eb935b Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 22:03:33 -0500 Subject: [PATCH 32/52] add verbose test output to macos build --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index fadbb4086..c82901991 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -32,4 +32,4 @@ jobs: build/muon -C build samu - name: test run: | - build/muon -C build test || true + build/muon -C build test -v -ddots || true From 77c756997988944990220f4c6613c0a30e9fdf23 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 22:23:51 -0500 Subject: [PATCH 33/52] fix bad job calculation when n = -1; --- src/platform/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/os.c b/src/platform/os.c index 9291d461f..8daae2bb7 100644 --- a/src/platform/os.c +++ b/src/platform/os.c @@ -15,7 +15,7 @@ os_parallel_job_count(void) int32_t n = os_ncpus(); if (n == -1) { - return -1; + return 4; } else if (n < 2) { return 2; } else { From ea04a9fd92789b5220d2d19852a28489e85329b7 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Sun, 14 Jan 2024 22:24:06 -0500 Subject: [PATCH 34/52] calculate ncpus properly on macos --- src/platform/posix/os.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/platform/posix/os.c b/src/platform/posix/os.c index 6deff3260..382b5d4ba 100644 --- a/src/platform/posix/os.c +++ b/src/platform/posix/os.c @@ -7,22 +7,31 @@ #include "compat.h" #include - -#ifdef MUON_HAVE_GETLOADAVG #include #include +// Messing with these defines and includes in the amalgamated build is +// difficult to reason about, so features requiring feature detection are +// disabled until we are bootstrapped. +#if defined(MUON_BOOTSTRAPPED) + #if defined(__APPLE__) // On macOS, getloadavg is unavailable if _POSIX_C_SOURCE is defined #undef _POSIX_C_SOURCE -#else -// Otherwise assume getloadavg is available when _BSD_SOURCE is defined +// for sysctl +#include +#include +#endif // defined(__APPLE__) + +#ifdef MUON_HAVE_GETLOADAVG +#if !defined(__APPLE__) +// Assume getloadavg is available when _BSD_SOURCE is defined #define _BSD_SOURCE #endif - #include +#endif // MUON_HAVE_GETLOADAVG -#endif +#endif // MUON_BOOTSTRAPPED #include "log.h" #include "platform/os.h" @@ -45,8 +54,17 @@ int os_getopt(int argc, char * const argv[], const char *optstring) int32_t os_ncpus(void) { -#ifdef _SC_NPROCESSORS_ONLN +#if defined(_SC_NPROCESSORS_ONLN) return sysconf(_SC_NPROCESSORS_ONLN); +#elif defined(__APPLE__) && defined(MUON_BOOTSTRAPPED) + int64_t res; + size_t size = sizeof(res); + int r = sysctlbyname("hw.activecpu", &res, &size, NULL, 0); + if (r == -1) { + return -1; + } else { + return res; + } #else return -1; #endif From 02890429e0261d78ea4185ca33ac048e89d777a5 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 17 Jan 2024 10:33:41 -0500 Subject: [PATCH 35/52] remove bogus assert --- src/external/samurai/util.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index 2bbfe40c0..ce614bb46 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -198,7 +198,6 @@ samu_xasprintf(struct samu_arena *a, char **s, const char *fmt, ...) va_start(ap, fmt); ret = vsnprintf(*s, n, fmt, ap); va_end(ap); - assert(ret < 0 || (size_t)ret >= n); assert(!(ret < 0 || (size_t)ret >= n)); return ret; From 0d9cc5ffbf86cf4a2ca0d86fd85e8d39ae90ea8a Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 17 Jan 2024 10:34:01 -0500 Subject: [PATCH 36/52] samu_makedirs: don't fail if dir already exists --- src/external/samurai/util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index ce614bb46..818057abf 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -284,13 +284,13 @@ samu_canonpath(struct samu_string *path) int samu_makedirs(struct samu_string *path, bool parent) { - bool ok; + bool ok = true; if (parent) { SBUF_manual(dirname); path_dirname(0, &dirname, path->s); ok = fs_mkdir_p(dirname.buf); sbuf_destroy(&dirname); - } else { + } else if (!fs_dir_exists(path->s)) { ok = fs_mkdir(path->s); } From e78d694c88e7b5df6f597a6556df52e3bff7c0f8 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 17 Jan 2024 10:34:39 -0500 Subject: [PATCH 37/52] samu deps.c: avoid rename() call --- src/external/samurai/deps.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/external/samurai/deps.c b/src/external/samurai/deps.c index 06e3862cd..7d72e61a7 100644 --- a/src/external/samurai/deps.c +++ b/src/external/samurai/deps.c @@ -12,6 +12,7 @@ #include #include "external/samurai/ctx.h" +#include "platform/filesystem.h" #include "external/samurai/build.h" #include "external/samurai/deps.h" @@ -51,7 +52,6 @@ specified previously in node records. #define SAMU_MAX_RECORD_SIZE (1 << 19) static const char ninja_depsname[] = ".ninja_deps"; -static const char ninja_depstmpname[] = ".ninja_deps.tmp"; static const char ninja_depsheader[] = "# ninjadeps\n"; static const uint32_t ninja_depsver = 4; @@ -107,7 +107,7 @@ samu_recorddeps(struct samu_ctx *ctx, struct samu_node *out, struct samu_nodearr void samu_depsinit(struct samu_ctx *ctx, const char *builddir) { - char *depspath = (char *)ninja_depsname, *depstmppath = (char *)ninja_depstmpname; + char *depspath = (char *)ninja_depsname; uint32_t *buf, cap, ver, sz, id; size_t len, i, j, nrecord; bool isdep; @@ -128,12 +128,9 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) buf = samu_xmalloc(&ctx->arena, cap); if (builddir) samu_xasprintf(&ctx->arena, &depspath, "%s/%s", builddir, ninja_depsname); - ctx->deps.depsfile = fopen(depspath, "r+"); - if (!ctx->deps.depsfile) { - if (errno != ENOENT) - samu_fatal("open %s:", depspath); + if (!fs_exists(depspath)) goto rewrite; - } + ctx->deps.depsfile = fs_fopen(depspath, "r+"); if (!fgets((char *)buf, sizeof(ninja_depsheader), ctx->deps.depsfile)) goto rewrite; if (strcmp((char *)buf, ninja_depsheader) != 0) { @@ -240,11 +237,9 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) fclose(ctx->deps.depsfile); ctx->deps.depsfile = NULL; } - if (builddir) - samu_xasprintf(&ctx->arena, &depstmppath, "%s/%s", builddir, ninja_depstmpname); - ctx->deps.depsfile = fopen(depstmppath, "w"); + ctx->deps.depsfile = fopen(depspath, "w"); if (!ctx->deps.depsfile) - samu_fatal("open %s:", depstmppath); + samu_fatal("open %s:", depspath); samu_depswrite(ctx, ninja_depsheader, 1, sizeof(ninja_depsheader) - 1); samu_depswrite(ctx, &ninja_depsver, 1, sizeof(ninja_depsver)); @@ -270,8 +265,6 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) fflush(ctx->deps.depsfile); if (ferror(ctx->deps.depsfile)) samu_fatal("deps log write failed"); - if (rename(depstmppath, depspath) < 0) - samu_fatal("deps log rename:"); } void From 05b480de39411bf36578dcbe6fc8ce1d3e6d3562 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 17 Jan 2024 10:35:09 -0500 Subject: [PATCH 38/52] fs_read: report EOF error correctly --- src/platform/filesystem.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/platform/filesystem.c b/src/platform/filesystem.c index 80fb58680..ea68e0542 100644 --- a/src/platform/filesystem.c +++ b/src/platform/filesystem.c @@ -311,7 +311,9 @@ fs_fread(void *ptr, size_t size, FILE *f) if (r == size) { return true; } else { - if ((err = ferror(f))) { + if (feof(f)) { + LOG_E("fread got EOF"); + } else if ((err = ferror(f))) { LOG_E("fread failed: %s", strerror(err)); } else { LOG_E("fread failed: unknown"); From efaff852b1fb699be129462db7bf2f25cc30ad6d Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 17 Jan 2024 10:36:30 -0500 Subject: [PATCH 39/52] change muon private dir to .muon --- src/backend/output.c | 2 +- src/main.c | 9 +++++++-- tests/project/common/44 pkgconfig-gen/meson.build | 2 +- tests/project/runner.sh | 6 ++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/backend/output.c b/src/backend/output.c index 0e0ffa5b0..b19211582 100644 --- a/src/backend/output.c +++ b/src/backend/output.c @@ -13,7 +13,7 @@ #include "tracy.h" const struct output_path output_path = { - .private_dir = "muon-private", + .private_dir = ".muon", .summary = "summary.txt", .tests = "tests.dat", .install = "install.dat", diff --git a/src/main.c b/src/main.c index f74cca293..357cfbecf 100644 --- a/src/main.c +++ b/src/main.c @@ -12,6 +12,7 @@ #include "args.h" #include "backend/backend.h" +#include "backend/output.h" #include "cmd_install.h" #include "cmd_test.h" #include "embedded.h" @@ -39,7 +40,7 @@ static bool ensure_in_build_dir(void) { - if (!fs_dir_exists("muon-private")) { + if (!fs_dir_exists(output_path.private_dir)) { LOG_E("this subcommand must be run from a build directory"); return false; } @@ -363,9 +364,12 @@ cmd_summary(uint32_t argc, uint32_t argi, char *const argv[]) return false; } + SBUF_manual(path); + path_join(0, &path, output_path.private_dir, output_path.summary); + bool ret = false; struct source src = { 0 }; - if (!fs_read_entire_file("muon-private/summary.txt", &src)) { + if (!fs_read_entire_file(path.buf, &src)) { goto ret; } @@ -373,6 +377,7 @@ cmd_summary(uint32_t argc, uint32_t argi, char *const argv[]) ret = true; ret: + sbuf_destroy(&path); fs_source_destroy(&src); return ret; } diff --git a/tests/project/common/44 pkgconfig-gen/meson.build b/tests/project/common/44 pkgconfig-gen/meson.build index 5ac9be6ff..bf5d03b2f 100644 --- a/tests/project/common/44 pkgconfig-gen/meson.build +++ b/tests/project/common/44 pkgconfig-gen/meson.build @@ -195,7 +195,7 @@ to_validate += 'libvartest2' cmp = find_program('diff') -privder = 'muon-private' +privder = '.muon' env = environment() env.prepend('PKG_CONFIG_PATH', meson.current_build_dir() / privder) diff --git a/tests/project/runner.sh b/tests/project/runner.sh index 4ca7f04a0..02d5677a2 100755 --- a/tests/project/runner.sh +++ b/tests/project/runner.sh @@ -4,6 +4,8 @@ set -eux +muon_private=".muon" + muon="$1" ninja="$2" source="$3" @@ -20,8 +22,8 @@ if [ $git_clean -eq 1 ]; then git clean -xdf -- "$source" fi -mkdir -p "$build/muon-private" -log="$build/muon-private/build_log.txt" +mkdir -p "$build/$muon_private" +log="$build/$muon_private/build_log.txt" if [ $skip_analyze -eq 0 ]; then set +e From 3315256713cc7eaa67067dbd760645156764f47a Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 17 Jan 2024 10:36:49 -0500 Subject: [PATCH 40/52] set ninja builddir as muon private dir This causes .ninja* files to be put into .muon which slightly declutters the build dir. Not that anyone cares :) --- src/backend/ninja/rules.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/ninja/rules.c b/src/backend/ninja/rules.c index ce593b6df..7bc9177d3 100644 --- a/src/backend/ninja/rules.c +++ b/src/backend/ninja/rules.c @@ -280,8 +280,10 @@ ninja_write_rules(FILE *out, struct workspace *wk, struct project *main_proj, out, "# This is the build file for project \"%s\"\n" "# It is autogenerated by the muon build system.\n" - "ninja_required_version = 1.7.1\n\n", - get_cstr(wk, main_proj->cfg.name) + "ninja_required_version = 1.7.1\n" + "builddir = %s\n\n", + get_cstr(wk, main_proj->cfg.name), + output_path.private_dir ); if (machine_system() == machine_system_windows) { From 6089e9f6892a667d1c70786f6bfed461ebed3293 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:53:47 -0500 Subject: [PATCH 41/52] add win32_fatal --- include/platform/windows/win32_error.h | 3 +++ src/platform/windows/win32_error.c | 22 +++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/include/platform/windows/win32_error.h b/include/platform/windows/win32_error.h index 90b5add79..0ae51ac91 100644 --- a/include/platform/windows/win32_error.h +++ b/include/platform/windows/win32_error.h @@ -7,6 +7,9 @@ #ifndef MUON_PLATFORM_WINDOWS_WIN32_ERROR_H #define MUON_PLATFORM_WINDOWS_WIN32_ERROR_H +#include "compat.h" + const char *win32_error(void); +void win32_fatal(const char *fmt, ...) MUON_ATTR_FORMAT(printf, 1, 2); #endif diff --git a/src/platform/windows/win32_error.c b/src/platform/windows/win32_error.c index a379d381c..23e1421b1 100644 --- a/src/platform/windows/win32_error.c +++ b/src/platform/windows/win32_error.c @@ -7,19 +7,20 @@ #include "compat.h" #ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN #endif #include #define STRSAFE_NO_CB_FUNCTIONS #include #include "platform/windows/win32_error.h" - -static char _msg[4096]; +#include "log.h" const char * win32_error(void) { + static char _msg[4096]; + LPTSTR msg; DWORD err; @@ -36,3 +37,18 @@ win32_error(void) return _msg; } + +void +win32_fatal(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + log_plainv(fmt, ap); + va_end(ap); + if (fmt[strlen(fmt) - 1] == ':') { + log_plain(" %s", win32_error()); + } + log_plain("\n"); + exit(1); +} From 679fd8d89ece97e10356fa6485dc5e45e690b9c0 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:54:08 -0500 Subject: [PATCH 42/52] change samu mtime special values 1 and 2 seem more unlikely to me, and they are much more easily identified in a debugger --- include/external/samurai/graph.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/external/samurai/graph.h b/include/external/samurai/graph.h index 2214b3358..5fd427730 100644 --- a/include/external/samurai/graph.h +++ b/include/external/samurai/graph.h @@ -14,9 +14,9 @@ struct samu_string; /* set in the tv_nsec field of a node's mtime */ enum { /* we haven't stat the file yet */ - SAMU_MTIME_UNKNOWN = -1, + SAMU_MTIME_UNKNOWN = 1, /* the file does not exist */ - SAMU_MTIME_MISSING = -2, + SAMU_MTIME_MISSING = 2, }; void samu_graphinit(struct samu_ctx *ctx); From 267c9a8688546a1668c227608bfa3d9bb5dd5d59 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:55:44 -0500 Subject: [PATCH 43/52] refactor samu deps parser to not use f* calls --- src/external/samurai/deps.c | 295 +++++++++++++++++++++++------------- 1 file changed, 190 insertions(+), 105 deletions(-) diff --git a/src/external/samurai/deps.c b/src/external/samurai/deps.c index 7d72e61a7..e243462a7 100644 --- a/src/external/samurai/deps.c +++ b/src/external/samurai/deps.c @@ -11,6 +11,7 @@ #include #include +#include "log.h" #include "external/samurai/ctx.h" #include "platform/filesystem.h" @@ -21,32 +22,32 @@ #include "external/samurai/util.h" /* -.ninja_deps file format - -The header identifying the format is the string "# ninjadeps\n", followed by a -4-byte integer specifying the format version. After this is a series of binary -records. All integers in .ninja_deps are written in system byte-order. - -A record starts with a 4-byte integer indicating the record type and size. If -the high bit is set, then it is a dependency record. Otherwise, it is a node -record. In either case, the remaining 31 bits specify the size in bytes of the -rest of the record. The size must be a multiple of 4, and no larger than than -2^19. - -Node records are given in incrementing ID order, and must be given before any -dependency record that refers to it. The last 4-byte integer in the record is -used as a checksum to prevent corruption. Counting from 0, the n-th node record -(specifying the node with ID n) will have a checksum of ~n (bitwise negation of -n). The remaining bytes of the record specify the path of the node, padded with -NUL bytes to the next 4-byte boundary (start of the checksum value). - -A dependency record contains a list of dependencies for the edge that built a -particular node. The first 4-byte integer is the node ID. The second and third -4-byte integers are the low and high 32-bits of the UNIX mtime (in nanoseconds) -of the node when it was built. Following this is a sequence of 4-byte integers -specifying the IDs of the dependency nodes for this edge, which will have been -specified previously in node records. -*/ + .ninja_deps file format + + The header identifying the format is the string "# ninjadeps\n", followed by a + 4-byte integer specifying the format version. After this is a series of binary + records. All integers in .ninja_deps are written in system byte-order. + + A record starts with a 4-byte integer indicating the record type and size. If + the high bit is set, then it is a dependency record. Otherwise, it is a node + record. In either case, the remaining 31 bits specify the size in bytes of the + rest of the record. The size must be a multiple of 4, and no larger than than + 2^19. + + Node records are given in incrementing ID order, and must be given before any + dependency record that refers to it. The last 4-byte integer in the record is + used as a checksum to prevent corruption. Counting from 0, the n-th node record + (specifying the node with ID n) will have a checksum of ~n (bitwise negation of + n). The remaining bytes of the record specify the path of the node, padded with + NUL bytes to the next 4-byte boundary (start of the checksum value). + + A dependency record contains a list of dependencies for the edge that built a + particular node. The first 4-byte integer is the node ID. The second and third + 4-byte integers are the low and high 32-bits of the UNIX mtime (in nanoseconds) + of the node when it was built. Following this is a sequence of 4-byte integers + specifying the IDs of the dependency nodes for this edge, which will have been + specified previously in node records. + */ /* maximum record size (in bytes) */ #define SAMU_MAX_RECORD_SIZE (1 << 19) @@ -58,8 +59,9 @@ static const uint32_t ninja_depsver = 4; static void samu_depswrite(struct samu_ctx *ctx, const void *p, size_t n, size_t m) { - if (fwrite(p, n, m, ctx->deps.depsfile) != m) + if (fwrite(p, n, m, ctx->deps.depsfile) != m) { samu_fatal("deps log write:"); + } } static bool @@ -67,17 +69,20 @@ samu_recordid(struct samu_ctx *ctx, struct samu_node *n) { uint32_t sz, chk; - if (n->id != -1) + if (n->id != -1) { return false; - if (ctx->deps.entrieslen == INT32_MAX) + } + if (ctx->deps.entrieslen == INT32_MAX) { samu_fatal("too many nodes"); + } n->id = ctx->deps.entrieslen++; sz = (n->path->n + 7) & ~3; - if (sz + 4 >= SAMU_MAX_RECORD_SIZE) + if (sz + 4 >= SAMU_MAX_RECORD_SIZE) { samu_fatal("ID record too large"); + } samu_depswrite(ctx, &sz, 4, 1); samu_depswrite(ctx, n->path->s, 1, n->path->n); - samu_depswrite(ctx, (char[4]){0}, 1, sz - n->path->n - 4); + samu_depswrite(ctx, (char[4]){ 0 }, 1, sz - n->path->n - 4); chk = ~n->id; samu_depswrite(ctx, &chk, 4, 1); @@ -91,8 +96,9 @@ samu_recorddeps(struct samu_ctx *ctx, struct samu_node *out, struct samu_nodearr size_t i; sz = 12 + deps->len * 4; - if (sz + 4 >= SAMU_MAX_RECORD_SIZE) + if (sz + 4 >= SAMU_MAX_RECORD_SIZE) { samu_fatal("deps record too large"); + } sz |= 0x80000000; samu_depswrite(ctx, &sz, 4, 1); samu_depswrite(ctx, &out->id, 4, 1); @@ -100,8 +106,46 @@ samu_recorddeps(struct samu_ctx *ctx, struct samu_node *out, struct samu_nodearr samu_depswrite(ctx, &m, 4, 1); m = (mtime >> 32) & 0xffffffff; samu_depswrite(ctx, &m, 4, 1); - for (i = 0; i < deps->len; ++i) + for (i = 0; i < deps->len; ++i) { samu_depswrite(ctx, &deps->node[i]->id, 4, 1); + } +} + +struct seekable_source { + struct source src; + uint64_t i; +}; + +static size_t +src_fread(void *buf, size_t sz, size_t n, struct seekable_source *src) +{ + if (src->i >= src->src.len) { + return 0; + } + + size_t l = n, + r = (src->src.len - src->i) / sz; + r = r < l ? r : l; + + memcpy(buf, &src->src.src[src->i], r * sz); + src->i += r * sz; + return r; +} + +static char +src_getc(struct seekable_source *src) +{ + if (src->i >= src->src.len) { + return EOF; + } else { + char c = src->src.src[src->i]; + if (c == '\r' && src->src.src[src->i + 1] == '\n') { + c = '\n'; + ++src->i; + } + ++src->i; + return c; + } } void @@ -115,6 +159,8 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) struct samu_node *n; struct samu_edge *e; struct samu_entry *entry, *oldentries; + struct seekable_source src = { 0 }; + bool free_src = false; /* XXX: when ninja hits a bad record, it truncates the log to the last * good record. perhaps we should do the same. */ @@ -126,19 +172,27 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) ctx->deps.entrieslen = 0; cap = BUFSIZ; buf = samu_xmalloc(&ctx->arena, cap); - if (builddir) + if (builddir) { samu_xasprintf(&ctx->arena, &depspath, "%s/%s", builddir, ninja_depsname); - if (!fs_exists(depspath)) + } + if (!fs_exists(depspath)) { goto rewrite; - ctx->deps.depsfile = fs_fopen(depspath, "r+"); - if (!fgets((char *)buf, sizeof(ninja_depsheader), ctx->deps.depsfile)) + } + + if (!fs_read_entire_file(depspath, &src.src)) { + samu_warn("failed to read deps file"); goto rewrite; - if (strcmp((char *)buf, ninja_depsheader) != 0) { + } + free_src = true; + + if (strncmp(src.src.src, ninja_depsheader, strlen(ninja_depsheader)) != 0) { samu_warn("invalid deps log header"); goto rewrite; } - if (fread(&ver, sizeof(ver), 1, ctx->deps.depsfile) != 1) { - samu_warn(ferror(ctx->deps.depsfile) ? "deps log read:" : "deps log truncated"); + src.i += strlen(ninja_depsheader); + + if (src_fread(&ver, sizeof(ver), 1, &src) != 1) { + samu_warn("deps log truncated"); goto rewrite; } if (ver != ninja_depsver) { @@ -146,8 +200,9 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) goto rewrite; } for (nrecord = 0;; ++nrecord) { - if (fread(&sz, sizeof(sz), 1, ctx->deps.depsfile) != 1) + if (src_fread(&sz, sizeof(sz), 1, &src) != 1) { break; + } isdep = sz & 0x80000000; sz &= 0x7fffffff; if (sz > SAMU_MAX_RECORD_SIZE) { @@ -155,12 +210,13 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) goto rewrite; } if (sz > cap) { - do cap *= 2; - while (sz > cap); + do{ + cap *= 2; + }while (sz > cap); buf = samu_xmalloc(&ctx->arena, cap); } - if (fread(buf, sz, 1, ctx->deps.depsfile) != 1) { - samu_warn(ferror(ctx->deps.depsfile) ? "deps log read:" : "deps log truncated"); + if (src_fread(buf, sz, 1, &src) != 1) { + samu_warn("deps log truncated"); goto rewrite; } if (sz % 4) { @@ -181,8 +237,9 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) entry = &ctx->deps.entries[id]; entry->mtime = (int64_t)buf[2] << 32 | buf[1]; e = entry->node->gen; - if (!e || !samu_edgevar(ctx, e, "deps", true)) + if (!e || !samu_edgevar(ctx, e, "deps", true)) { continue; + } sz /= 4; entry->deps.len = sz; entry->deps.node = samu_xreallocarray(&ctx->arena, NULL, 0, sz, sizeof(n)); @@ -208,8 +265,9 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) goto rewrite; } len = sz - 4; - while (((char *)buf)[len - 1] == '\0') + while (((char *)buf)[len - 1] == '\0') { --len; + } path = samu_mkstr(&ctx->arena, len); memcpy(path->s, buf, len); path->s[len] = '\0'; @@ -221,31 +279,26 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) ctx->deps.entriescap = newcap; } n->id = ctx->deps.entrieslen; - ctx->deps.entries[ctx->deps.entrieslen++] = (struct samu_entry){.node = n}; + ctx->deps.entries[ctx->deps.entrieslen++] = (struct samu_entry){ .node = n }; } } - if (ferror(ctx->deps.depsfile)) { - samu_warn("deps log read:"); - goto rewrite; - } - if (nrecord <= 1000 || nrecord < 3 * ctx->deps.entrieslen) { - return; - } rewrite: if (ctx->deps.depsfile) { fclose(ctx->deps.depsfile); ctx->deps.depsfile = NULL; } - ctx->deps.depsfile = fopen(depspath, "w"); - if (!ctx->deps.depsfile) + ctx->deps.depsfile = fopen(depspath, "wb"); + if (!ctx->deps.depsfile) { samu_fatal("open %s:", depspath); + } samu_depswrite(ctx, ninja_depsheader, 1, sizeof(ninja_depsheader) - 1); samu_depswrite(ctx, &ninja_depsver, 1, sizeof(ninja_depsver)); /* reset ID for all current entries */ - for (i = 0; i < ctx->deps.entrieslen; ++i) + for (i = 0; i < ctx->deps.entrieslen; ++i) { ctx->deps.entries[i].node->id = -1; + } /* save a temporary copy of the old entries */ oldentries = samu_xreallocarray(&ctx->arena, NULL, 0, ctx->deps.entrieslen, sizeof(ctx->deps.entries[0])); memcpy(oldentries, ctx->deps.entries, ctx->deps.entrieslen * sizeof(ctx->deps.entries[0])); @@ -254,25 +307,33 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) ctx->deps.entrieslen = 0; for (i = 0; i < len; ++i) { entry = &oldentries[i]; - if (!entry->deps.len) + if (!entry->deps.len) { continue; + } samu_recordid(ctx, entry->node); ctx->deps.entries[entry->node->id] = *entry; - for (j = 0; j < entry->deps.len; ++j) + for (j = 0; j < entry->deps.len; ++j) { samu_recordid(ctx, entry->deps.node[j]); + } samu_recorddeps(ctx, entry->node, &entry->deps, entry->mtime); } fflush(ctx->deps.depsfile); - if (ferror(ctx->deps.depsfile)) + if (ferror(ctx->deps.depsfile)) { samu_fatal("deps log write failed"); + } + + if (free_src) { + fs_source_destroy(&src.src); + } } void samu_depsclose(struct samu_ctx *ctx) { fflush(ctx->deps.depsfile); - if (ferror(ctx->deps.depsfile)) + if (ferror(ctx->deps.depsfile)) { samu_fatal("deps log write failed"); + } fclose(ctx->deps.depsfile); ctx->deps.depsfile = NULL; } @@ -281,20 +342,25 @@ static struct samu_nodearray * samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) { struct samu_string *in, *out = NULL; - FILE *f; + struct seekable_source src = { 0 }; int c, n; bool sawcolon; ctx->deps.deps.len = 0; - f = fopen(name, "r"); - if (!f) { - if (errno == ENOENT && allowmissing) + if (!fs_exists(name)) { + if (allowmissing) { return &ctx->deps.deps; - return NULL; + } + return 0; + } + + if (!fs_read_entire_file(name, &src.src)) { + return 0; } + sawcolon = false; ctx->deps.buf.len = 0; - c = getc(f); + c = src_getc(&src); for (;;) { /* TODO: this parser needs to be rewritten to be made simpler */ while (isalnum(c) || strchr("$+,-./@\\_", c)) { @@ -303,14 +369,17 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) /* handle the crazy escaping generated by clang and gcc */ n = 0; do { - c = getc(f); - if (++n % 2 == 0) + c = src_getc(&src); + if (++n % 2 == 0) { samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + } } while (c == '\\'); - if ((c == ' ' || c == '\t') && n % 2 != 0) + if ((c == ' ' || c == '\t') && n % 2 != 0) { break; - for (; n > 2; n -= 2) + } + for (; n > 2; n -= 2) { samu_bufadd(&ctx->arena, &ctx->deps.buf, '\\'); + } switch (c) { case '#': break; case '\n': c = ' '; continue; @@ -318,19 +387,19 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) } break; case '$': - c = getc(f); + c = src_getc(&src); if (c != '$') { - samu_warn("bad depfile: contains variable reference"); + samu_warn("bad depfile[%lld]: contains variable reference", src.i); goto err; } break; } samu_bufadd(&ctx->arena, &ctx->deps.buf, c); - c = getc(f); + c = src_getc(&src); } if (sawcolon) { if (!isspace(c) && c != EOF) { - samu_warn("bad depfile: '%c' is not a valid target character", c); + samu_warn("bad depfile[%lld]: '%c' is not a valid target character", src.i, c); goto err; } if (ctx->deps.buf.len > 0) { @@ -346,16 +415,20 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) } if (c == '\n') { sawcolon = false; - do c = getc(f); - while (c == '\n'); + do{ + c = src_getc(&src); + }while (c == '\n'); } - if (c == EOF) + if (c == EOF) { break; + } } else { - while (isblank(c)) - c = getc(f); - if (c == EOF) + while (isblank(c)) { + c = src_getc(&src); + } + if (c == EOF) { break; + } if (c != ':') { samu_warn("bad depfile: expected ':', saw '%c'", c); goto err; @@ -365,34 +438,35 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) memcpy(out->s, ctx->deps.buf.data, ctx->deps.buf.len); out->s[ctx->deps.buf.len] = '\0'; } else if (out->n != ctx->deps.buf.len || memcmp(ctx->deps.buf.data, out->s, ctx->deps.buf.len) != 0) { - samu_warn("bad depfile: multiple outputs: %.*s != %s", (int)ctx->deps.buf.len, ctx->deps.buf.data, out->s); + samu_fatal("bad depfile: multiple outputs: %.*s != %s", (int)ctx->deps.buf.len, ctx->deps.buf.data, out->s); goto err; } sawcolon = true; - c = getc(f); + c = src_getc(&src); } ctx->deps.buf.len = 0; for (;;) { if (c == '\\') { - if (getc(f) != '\n') { - samu_warn("bad depfile: '\\' only allowed before newline"); + if (src_getc(&src) != '\n') { + samu_warn("bad depfile[%lld]: '\\' only allowed before newline", src.i); + printf("%s", src.src.src); goto err; } } else if (!isblank(c)) { break; } - c = getc(f); + c = src_getc(&src); } } - if (ferror(f)) { - samu_warn("depfile read:"); - goto err; - } - fclose(f); + + fs_source_destroy(&src.src); return &ctx->deps.deps; err: - fclose(f); + fs_source_destroy(&src.src); + if (!allowmissing) { + samu_fatal("failed to parse depfile %s", name); + } return NULL; } @@ -403,23 +477,27 @@ samu_depsload(struct samu_ctx *ctx, struct samu_edge *e) struct samu_nodearray *deps = NULL; struct samu_node *n; - if (e->flags & FLAG_DEPS) + if (e->flags & FLAG_DEPS) { return; + } e->flags |= FLAG_DEPS; n = e->out[0]; deptype = samu_edgevar(ctx, e, "deps", true); if (deptype) { - if (n->id != -1 && n->mtime <= ctx->deps.entries[n->id].mtime) + if (n->id != -1 && n->mtime <= ctx->deps.entries[n->id].mtime) { deps = &ctx->deps.entries[n->id].deps; - else if (ctx->buildopts.explain) + }else if (ctx->buildopts.explain) { samu_warn("explain %s: missing or outdated record in .ninja_deps", n->path->s); + } } else { depfile = samu_edgevar(ctx, e, "depfile", false); - if (!depfile) + if (!depfile) { return; + } deps = samu_depsparse(ctx, depfile->s, false); - if (ctx->buildopts.explain && !deps) + if (ctx->buildopts.explain && !deps) { samu_warn("explain %s: missing or invalid depfile", n->path->s); + } } if (deps) { samu_edgeadddeps(ctx, e, deps->node, deps->len); @@ -440,8 +518,9 @@ samu_depsrecord(struct samu_ctx *ctx, struct samu_edge *e) bool update; deptype = samu_edgevar(ctx, e, "deps", true); - if (!deptype || deptype->n == 0) + if (!deptype || deptype->n == 0) { return; + } if (strcmp(deptype->s, "gcc") != 0) { samu_warn("unsuported deps type: %s", deptype->s); return; @@ -453,31 +532,37 @@ samu_depsrecord(struct samu_ctx *ctx, struct samu_edge *e) } out = e->out[0]; deps = samu_depsparse(ctx, depfile->s, true); - if (!ctx->buildopts.keepdepfile) + if (!ctx->buildopts.keepdepfile) { remove(depfile->s); - if (!deps) + } + if (!deps) { return; + } update = false; entry = NULL; if (samu_recordid(ctx, out)) { update = true; } else { entry = &ctx->deps.entries[out->id]; - if (entry->mtime != out->mtime || entry->deps.len != deps->len) + if (entry->mtime != out->mtime || entry->deps.len != deps->len) { update = true; + } for (i = 0; i < deps->len && !update; ++i) { - if (entry->deps.node[i] != deps->node[i]) + if (entry->deps.node[i] != deps->node[i]) { update = true; + } } } for (i = 0; i < deps->len; ++i) { n = deps->node[i]; - if (samu_recordid(ctx, n)) + if (samu_recordid(ctx, n)) { update = true; + } } if (update) { samu_recorddeps(ctx, out, deps, out->mtime); - if (fflush(ctx->deps.depsfile) < 0) + if (fflush(ctx->deps.depsfile) < 0) { samu_fatal("deps log flush:"); + } } } From 9e34ab31437b1390e157abadaa297aad7dc5f896 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:56:14 -0500 Subject: [PATCH 44/52] refactor samu log parser to not use f* calls --- src/external/samurai/log.c | 258 +++++++++++++++++++++---------------- 1 file changed, 148 insertions(+), 110 deletions(-) diff --git a/src/external/samurai/log.c b/src/external/samurai/log.c index 617e0a338..5227a799f 100644 --- a/src/external/samurai/log.c +++ b/src/external/samurai/log.c @@ -12,154 +12,192 @@ #include #include "external/samurai/ctx.h" +#include "formats/lines.h" +#include "log.h" #include "external/samurai/graph.h" #include "external/samurai/log.h" #include "external/samurai/util.h" static const char *samu_logname = ".ninja_log"; -static const char *samu_logtmpname = ".ninja_log.tmp"; -static const char *samu_logfmt = "# ninja log v%d\n"; +static const char *samu_log_version_fmt = "# ninja log v%d\n"; static const int samu_logver = 5; -static char * -samu_nextfield(char **end) -{ - char *s = *end; - - if (!*s) { - samu_warn("corrupt build log: missing field"); - return NULL; - } - *end += strcspn(*end, "\t\n"); - if (**end) - *(*end)++ = '\0'; +enum samu_log_field { + samu_log_field_start_time, + samu_log_field_end_time, + samu_log_field_mtime, + samu_log_field_output_path, + samu_log_field_command_hash, + samu_log_field_count, +}; - return s; -} +struct samu_log_parse_ctx { + uint32_t line_no; + size_t nentry; + struct samu_ctx *samu_ctx; +}; -void -samu_loginit(struct samu_ctx *ctx, const char *builddir) +static enum iteration_result +samu_log_parse_cb(void *_ctx, char *line, size_t len) { - int ver; - char *logpath = (char *)samu_logname, *logtmppath = (char *)samu_logtmpname, *p, *s; - size_t nline, nentry, i; - struct samu_edge *e; + struct samu_log_parse_ctx *ctx = _ctx; + char *fields[samu_log_field_count] = { 0 }; struct samu_node *n; - int64_t mtime; - struct samu_buffer buf = {0}; - nline = 0; - nentry = 0; + if (ctx->line_no == 1) { + int ver; + if (sscanf(line, samu_log_version_fmt, &ver) < 1 || ver != samu_logver) { + return ir_done; + } - if (ctx->log.logfile) { - fclose(ctx->log.logfile); - ctx->log.logfile = NULL; + goto cont; } - if (builddir) - samu_xasprintf(&ctx->arena, &logpath, "%s/%s", builddir, samu_logname); - ctx->log.logfile = fopen(logpath, "r+"); - if (!ctx->log.logfile) { - if (errno != ENOENT) - samu_fatal("open %s:", logpath); - goto rewrite; + + { + char *p = line; + uint32_t i; + for (i = 0; i < samu_log_field_count; ++i) { + fields[i] = p; + if (!(p = strchr(p, '\t'))) { + break; + } + + *p = 0; + ++p; + } } - setvbuf(ctx->log.logfile, NULL, _IOLBF, 0); - if (fscanf(ctx->log.logfile, samu_logfmt, &ver) < 1) - goto rewrite; - if (ver != samu_logver) - goto rewrite; - - for (;;) { - if (buf.cap - buf.len < BUFSIZ) { - size_t newcap = buf.cap ? buf.cap * 2 : BUFSIZ; - buf.data = samu_xreallocarray(&ctx->arena, buf.data, buf.cap, newcap, 1); - buf.cap = newcap; + + { // get node + if (!fields[samu_log_field_output_path]) { + samu_warn("missing output path"); + goto corrupt_line; } - buf.data[buf.cap - 2] = '\0'; - if (!fgets(buf.data + buf.len, buf.cap - buf.len, ctx->log.logfile)) - break; - if (buf.data[buf.cap - 2] && buf.data[buf.cap - 2] != '\n') { - buf.len = buf.cap - 1; - continue; + + n = samu_nodeget(ctx->samu_ctx, fields[samu_log_field_output_path], 0); + if (!n || !n->gen) { + goto cont; } - ++nline; - p = buf.data; - buf.len = 0; - if (!samu_nextfield(&p)) /* start time */ - continue; - if (!samu_nextfield(&p)) /* end time */ - continue; - s = samu_nextfield(&p); /* mtime (used for restat) */ - if (!s) - continue; - mtime = strtoll(s, &s, 10); - if (*s) { - samu_warn("corrupt build log: invalid mtime"); - continue; + if (n->logmtime == SAMU_MTIME_MISSING) { + ++ctx->nentry; } - s = samu_nextfield(&p); /* output path */ - if (!s) - continue; - n = samu_nodeget(ctx, s, 0); - if (!n || !n->gen) - continue; - if (n->logmtime == SAMU_MTIME_MISSING) - ++nentry; - n->logmtime = mtime; - s = samu_nextfield(&p); /* command hash */ - if (!s) - continue; - n->hash = strtoull(s, &s, 16); - if (*s) { - samu_warn("corrupt build log: invalid hash for '%s'", n->path->s); - continue; + } + + { // get mtime + if (!fields[samu_log_field_mtime]) { + samu_warn("missing mtime"); + goto corrupt_line; + } + + char *endptr; + n->logmtime = strtoll(fields[samu_log_field_mtime], &endptr, 10); + if (*endptr) { + samu_warn("invalid mtime: %s", fields[samu_log_field_mtime]); + goto corrupt_line; } } - if (ferror(ctx->log.logfile)) { - samu_warn("build log read:"); - goto rewrite; + + { // get output hash + if (!fields[samu_log_field_command_hash]) { + samu_warn("missing command hash"); + goto corrupt_line; + } + + char *endptr; + n->hash = strtoull(fields[samu_log_field_command_hash], &endptr, 16); + if (*endptr) { + samu_warn("invalid hash for '%s'", n->path->s); + goto corrupt_line; + } } - if (nline <= 100 || nline <= 3 * nentry) { - return; + +cont: + ++ctx->line_no; + return ir_cont; +corrupt_line: + samu_warn("corrupt build log @ line %d", ctx->line_no); + goto cont; +} + +static void +samu_log_open_and_write(struct samu_ctx *ctx, const char *builddir, bool write_graph) +{ + const struct samu_edge *e; + struct samu_node *n; + uint32_t i; + + const char *logpath; + if (builddir) { + char *path; + samu_xasprintf(&ctx->arena, &path, "%s/%s", builddir, samu_logname); + logpath = path; + } else { + logpath = samu_logname; } -rewrite: - if (ctx->log.logfile) { - fclose(ctx->log.logfile); - ctx->log.logfile = NULL; + if (!(ctx->log.logfile = fs_fopen(logpath, "wb"))) { + samu_fatal("open %s", logpath); + return; } - if (builddir) - samu_xasprintf(&ctx->arena, &logtmppath, "%s/%s", builddir, samu_logtmpname); - ctx->log.logfile = fopen(logtmppath, "w"); - if (!ctx->log.logfile) - samu_fatal("open %s:", logtmppath); - setvbuf(ctx->log.logfile, NULL, _IOLBF, 0); - fprintf(ctx->log.logfile, samu_logfmt, samu_logver); - if (nentry > 0) { + + fprintf(ctx->log.logfile, samu_log_version_fmt, samu_logver); + + if (write_graph) { for (e = ctx->graph.alledges; e; e = e->allnext) { for (i = 0; i < e->nout; ++i) { n = e->out[i]; - if (!n->hash) + if (!n->hash) { continue; + } samu_logrecord(ctx, n); } } } - fflush(ctx->log.logfile); - if (ferror(ctx->log.logfile)) - samu_fatal("build log write failed"); - if (rename(logtmppath, logpath) < 0) - samu_fatal("build log rename:"); +} + +void +samu_loginit(struct samu_ctx *ctx, const char *builddir) +{ + char *logpath = (char *)samu_logname; + + if (ctx->log.logfile) { + samu_logclose(ctx); + } + + if (builddir) { + samu_xasprintf(&ctx->arena, &logpath, "%s/%s", builddir, samu_logname); + } + + if (!fs_exists(logpath)) { + samu_log_open_and_write(ctx, builddir, false); + return; + } + + struct source src = { 0 }; + if (!fs_read_entire_file(logpath, &src)) { + samu_fatal("failed to read log file at %s", logpath); + } + + struct samu_log_parse_ctx samu_log_parse_ctx = { + .line_no = 1, + .samu_ctx = ctx, + }; + + each_line((char *)src.src, src.len, &samu_log_parse_ctx, samu_log_parse_cb); + + fs_source_destroy(&src); + + /* if (samu_log_parse_ctx.line_no <= 100 || samu_log_parse_ctx.line_no <= 3 * samu_log_parse_ctx.nentry) { */ + /* return; */ + /* } */ + + samu_log_open_and_write(ctx, builddir, true); } void samu_logclose(struct samu_ctx *ctx) { - fflush(ctx->log.logfile); - if (ferror(ctx->log.logfile)) - samu_fatal("build log write failed"); - fclose(ctx->log.logfile); + fs_fclose(ctx->log.logfile); ctx->log.logfile = NULL; } From 2aabf8d18703d3b1b0d5955398ac7bf0093edebc Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:56:27 -0500 Subject: [PATCH 45/52] remove setvbuf call --- src/external/samurai/samu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/external/samurai/samu.c b/src/external/samurai/samu.c index 4246836df..798a0cac4 100644 --- a/src/external/samurai/samu.c +++ b/src/external/samurai/samu.c @@ -224,8 +224,6 @@ samu_main(int argc, char *argv[], struct samu_opts *opts) if (!ctx->buildopts.statusfmt) ctx->buildopts.statusfmt = "[%s/%t] "; - setvbuf(stdout, NULL, _IOLBF, 0); - tries = 0; retry: /* (re-)initialize global graph, environment, and parse structures */ From 548c53edcbdb0612e1506849c6c8030bff6d4fc0 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:56:57 -0500 Subject: [PATCH 46/52] avoid non-standard %z specifier --- src/external/samurai/util.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index 818057abf..e4e643d7d 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -6,7 +6,6 @@ #include "compat.h" -#include #include #include #include @@ -121,7 +120,7 @@ samu_arena_destroy(struct samu_arena *a) z_free(a->blocks[i]); } - L("samu allocd %zd blocks, a:%zd, f:%zd, r:%3.3f\n", a->blocks_len, a->allocd, a->filled, (float)a->filled / (float)a->allocd * 100.0f); + L("samu allocd %d blocks, a:%d, f:%d, r:%3.3f\n", a->blocks_len, a->allocd, a->filled, (float)a->filled / (float)a->allocd * 100.0f); z_free(a->blocks); } From 2882acb4ecbc188ea907bcdad3014a6a2d237cde Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:57:34 -0500 Subject: [PATCH 47/52] sbuf: allow into_str with manual alloc'd sbuf --- src/lang/string.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lang/string.c b/src/lang/string.c index f98a065c1..c8e49ab49 100644 --- a/src/lang/string.c +++ b/src/lang/string.c @@ -726,15 +726,16 @@ sbuf_pushf(struct workspace *wk, struct sbuf *sb, const char *fmt, ...) obj sbuf_into_str(struct workspace *wk, struct sbuf *sb) { - assert(!(sb->flags & sbuf_flag_string_exposed) - && !(sb->flags & sbuf_flag_overflow_alloc)); + assert(!(sb->flags & sbuf_flag_string_exposed)); - if (sb->flags & sbuf_flag_overflown) { + if (!(sb->flags & sbuf_flag_overflow_alloc) && sb->flags & sbuf_flag_overflown) { sb->flags |= sbuf_flag_string_exposed; struct str *ss = (struct str *)get_str(wk, sb->s); assert(strlen(sb->buf) == sb->len); ss->len = sb->len; return sb->s; + } else if (!sb->len) { + return make_str(wk, ""); } else { return make_strn(wk, sb->buf, sb->len); } From 7e01271a64634955e223bf18b7ab23ecd70b4d7d Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:58:10 -0500 Subject: [PATCH 48/52] use sbuf_into_str with run_cmd output --- src/functions/kernel.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/functions/kernel.c b/src/functions/kernel.c index cb3eebd7e..b7849c333 100644 --- a/src/functions/kernel.c +++ b/src/functions/kernel.c @@ -676,8 +676,8 @@ find_program(struct workspace *wk, struct find_program_iter_ctx *ctx, obj prog) /* 7. [provide] sections in subproject wrap files, if wrap_mode is set to anything other than nofallback */ if (t == obj_string - && wrap_mode != wrap_mode_nofallback - && ctx->requirement == requirement_required) { + && wrap_mode != wrap_mode_nofallback + && ctx->requirement == requirement_required) { if (!find_program_check_fallback(wk, ctx, prog)) { return false; } @@ -1080,8 +1080,8 @@ func_run_command(struct workspace *wk, obj _, uint32_t args_node, obj *res) run_result->out = make_str(wk, ""); run_result->err = make_str(wk, ""); } else { - run_result->out = make_strn(wk, cmd_ctx.out.buf, cmd_ctx.out.len); - run_result->err = make_strn(wk, cmd_ctx.err.buf, cmd_ctx.err.len); + run_result->out = sbuf_into_str(wk, &cmd_ctx.out); + run_result->err = sbuf_into_str(wk, &cmd_ctx.err); } ret = true; From e147af2afc1b1b84caeb3010abd4639cb34f8021 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:58:38 -0500 Subject: [PATCH 49/52] fix implementation of windows fs_mtime --- src/external/samurai/graph.c | 4 ++++ src/platform/windows/filesystem.c | 16 ++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/external/samurai/graph.c b/src/external/samurai/graph.c index bb539e4ee..ae4c01493 100644 --- a/src/external/samurai/graph.c +++ b/src/external/samurai/graph.c @@ -6,10 +6,12 @@ #include "compat.h" +#include #include #include #include "external/samurai/ctx.h" +#include "log.h" #include "platform/filesystem.h" #include "external/samurai/env.h" @@ -79,6 +81,8 @@ samu_nodestat(struct samu_node *n) case fs_mtime_result_err: samu_fatal("stat %s:", n->path->s); return; + default: + assert(false && "unreachable"); } } diff --git a/src/platform/windows/filesystem.c b/src/platform/windows/filesystem.c index 18d41d678..73c92aba4 100644 --- a/src/platform/windows/filesystem.c +++ b/src/platform/windows/filesystem.c @@ -7,11 +7,12 @@ #include "compat.h" #ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN #endif #include #include #include +#include #include "log.h" #include "lang/string.h" @@ -339,7 +340,7 @@ fs_is_a_tty_from_fd(int fd) /* * Check the name of the pipe: * '\{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master' - */ + */ p = fni->FileName; if (is_wprefix(p, L"\\cygwin-")) { p += 8; @@ -351,7 +352,7 @@ fs_is_a_tty_from_fd(int fd) if (p) { /* Skip 16-digit hexadecimal. */ if (wcsspn(p, L"0123456789abcdefABCDEF") == 16) { - p+= 16; + p += 16; } else { p = NULL; } @@ -359,8 +360,7 @@ fs_is_a_tty_from_fd(int fd) if (p) { if (is_wprefix(p, L"-pty")) { p += 4; - } - else { + }else { p = NULL; } } @@ -377,8 +377,7 @@ fs_is_a_tty_from_fd(int fd) //p += 12; } else if (is_wprefix(p, L"-to-master")) { //p += 10; - } - else { + }else { p = NULL; } } @@ -506,5 +505,6 @@ fs_mtime(const char *path, int64_t *mtime) t.LowPart = d.ftLastWriteTime.dwLowDateTime; t.HighPart = d.ftLastWriteTime.dwHighDateTime; - return t.QuadPart * 100; + *mtime = t.QuadPart * 100; + return fs_mtime_result_ok; } From 3e1f9a196c01cc34a36014efd44fd24c7a703234 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Wed, 24 Jan 2024 20:59:17 -0500 Subject: [PATCH 50/52] refactor windows run_cmd to allow async commands also add run_cmd_unsplit for samu on windows --- include/platform/run_cmd.h | 44 ++-- src/external/samurai/build.c | 16 +- src/platform/posix/run_cmd.c | 9 +- src/platform/windows/run_cmd.c | 437 +++++++++++++++++++-------------- 4 files changed, 305 insertions(+), 201 deletions(-) diff --git a/include/platform/run_cmd.h b/include/platform/run_cmd.h index 8893e5945..ca33c998c 100644 --- a/include/platform/run_cmd.h +++ b/include/platform/run_cmd.h @@ -9,24 +9,20 @@ #include #include #ifdef _WIN32 -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif -#include + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #include #else -#include + #include #endif +#include "lang/string.h" + enum run_cmd_state { - run_cmd_error, run_cmd_running, run_cmd_finished, -}; - -struct run_cmd_pipe_ctx { - size_t size; - size_t len; - char *buf; + run_cmd_error, }; enum run_cmd_ctx_flags { @@ -34,8 +30,19 @@ enum run_cmd_ctx_flags { run_cmd_ctx_flag_dont_capture = 1 << 1, }; +#ifdef _WIN32 +struct win_pipe_inst { + OVERLAPPED overlapped; + HANDLE handle; + HANDLE child_handle; + HANDLE event; + char overlapped_buf[4 << 10]; + bool is_pending, is_eof; +}; +#endif + struct run_cmd_ctx { - struct run_cmd_pipe_ctx err, out; + struct sbuf err, out; const char *err_msg; // set on error const char *chdir; // set by caller const char *stdin_path; // set by caller @@ -44,10 +51,7 @@ struct run_cmd_ctx { #ifdef _WIN32 HANDLE process; bool close_pipes; - struct { - OVERLAPPED overlap; - HANDLE pipe[2]; - } pipe_out, pipe_err; + struct win_pipe_inst pipe_out, pipe_err; #else int pipefd_out[2], pipefd_err[2]; int input_fd; @@ -71,4 +75,10 @@ bool run_cmd_argv(struct run_cmd_ctx *ctx, char *const *argv, const char *envstr enum run_cmd_state run_cmd_collect(struct run_cmd_ctx *ctx); void run_cmd_ctx_destroy(struct run_cmd_ctx *ctx); bool run_cmd_kill(struct run_cmd_ctx *ctx, bool force); + +// runs a command by passing a single string containing both the command and +// arguments. +// currently only used by samurai on windows +bool run_cmd_unsplit(struct run_cmd_ctx *ctx, char *cmd, const char *envstr, uint32_t envc); + #endif diff --git a/src/external/samurai/build.c b/src/external/samurai/build.c index 3d4f7ebdf..38b404cd7 100644 --- a/src/external/samurai/build.c +++ b/src/external/samurai/build.c @@ -9,6 +9,7 @@ #include #include "external/samurai/ctx.h" +#include "functions/machine.h" #include "log.h" #include "platform/os.h" #include "platform/run_cmd.h" @@ -264,7 +265,6 @@ samu_jobstart(struct samu_ctx *ctx, struct samu_job *j, struct samu_edge *e) size_t i; struct samu_node *n; struct samu_string *rspfile, *content; - char *argv[] = {"/bin/sh", "-c", NULL, NULL}; ++ctx->build.nstarted; for (i = 0; i < e->nout; ++i) { @@ -292,12 +292,19 @@ samu_jobstart(struct samu_ctx *ctx, struct samu_job *j, struct samu_edge *e) j->cmd_ctx.flags |= run_cmd_ctx_flag_dont_capture; } - argv[2] = j->cmd->s; - if (!ctx->build.consoleused) samu_printstatus(ctx, e, j->cmd); - if (!run_cmd_argv(&j->cmd_ctx, argv, 0, 0)) { + + bool cmd_started = false; + if (machine_system() == machine_system_windows) { + cmd_started = run_cmd_unsplit(&j->cmd_ctx, j->cmd->s, 0, 0); + } else { + char *argv[] = {"/bin/sh", "-c", j->cmd->s, NULL}; + cmd_started = run_cmd_argv(&j->cmd_ctx, argv, 0, 0); + } + + if (!cmd_started) { samu_warn("failed to start job: %s", j->cmd_ctx.err_msg); j->failed = true; return false; @@ -481,6 +488,7 @@ samu_build(struct samu_ctx *ctx) } if (numjobs == 0) break; + for (i = 0; i < jobslen; ++i) { if (!jobs[i].running) { continue; diff --git a/src/platform/posix/run_cmd.c b/src/platform/posix/run_cmd.c index 377beeefd..0831e06b3 100644 --- a/src/platform/posix/run_cmd.c +++ b/src/platform/posix/run_cmd.c @@ -82,8 +82,7 @@ copy_pipes(struct run_cmd_ctx *ctx) case copy_pipe_result_failed: return copy_pipe_result_failed; default: - assert(false && "unreachable"); - return copy_pipe_result_failed; + UNREACHABLE_RETURN; } } @@ -525,3 +524,9 @@ run_cmd_kill(struct run_cmd_ctx *ctx, bool force) return true; } + +bool +run_cmd_unsplit(struct run_cmd_ctx *ctx, char *cmd, const char *envstr, uint32_t envc) +{ + assert(false && "this function should only be called under windows"); +} diff --git a/src/platform/windows/run_cmd.c b/src/platform/windows/run_cmd.c index e7846e650..31a166bbb 100644 --- a/src/platform/windows/run_cmd.c +++ b/src/platform/windows/run_cmd.c @@ -7,7 +7,6 @@ #include "compat.h" #include -#include #include #define STRSAFE_NO_CB_FUNCTIONS @@ -15,21 +14,51 @@ #include "args.h" #include "buf_size.h" +#include "error.h" #include "log.h" #include "platform/filesystem.h" #include "platform/mem.h" #include "platform/path.h" #include "platform/run_cmd.h" +#include "platform/timer.h" #include "platform/windows/win32_error.h" -#define CLOSE_PIPE(p_) do { \ - if ((p_) != INVALID_HANDLE_VALUE && !CloseHandle(p_)) { \ - LOG_E("failed to close pipe: %s", win32_error()); \ - } \ - p_ = INVALID_HANDLE_VALUE; \ -} while (0) +static uint32_t cnt_open; -#define COPY_PIPE_BLOCK_SIZE BUF_SIZE_1k +#define record_handle(__h, v) _record_handle(__h, v, #__h) + +static bool +_record_handle(HANDLE *h, HANDLE v, const char *desc) +{ + if (v == INVALID_HANDLE_VALUE) { + return false; + } + + ++cnt_open; + *h = v; + return true; +} + +#define close_handle(__h) _close_handle(__h, #__h) + +static bool +_close_handle(HANDLE *h, const char *desc) +{ + if (*h == INVALID_HANDLE_VALUE) { + return true; + } + + assert(cnt_open); + + if (!CloseHandle(*h)) { + LOG_E("failed to close handle %s:%p: %s", desc, *h, win32_error()); + return false; + } + + --cnt_open; + *h = INVALID_HANDLE_VALUE; + return true; +} enum copy_pipe_result { copy_pipe_result_finished, @@ -37,102 +66,103 @@ enum copy_pipe_result { copy_pipe_result_failed, }; -/* - * [X] copy_pipe() - * [X] copy_pipes() - * [X] run_cmd_ctx_close_pipes() - * [X] run_cmd_collect() - * [X] open_run_cmd_pipe() - * [X] run_cmd_internal() - * [X] run_cmd_argv() - * [X] run_cmd() - * [X] run_cmd_ctx_destroy() - * [ ] run_cmd_kill() - */ - static enum copy_pipe_result -copy_pipe(HANDLE pipe, struct run_cmd_pipe_ctx *ctx) +copy_pipe(struct win_pipe_inst *pipe, struct sbuf *sbuf) { - char *buf; - char *it1; - char *it2; - size_t len; - DWORD bytes_read; - BOOL res; - - if (!ctx->size) { - ctx->size = COPY_PIPE_BLOCK_SIZE; - ctx->len = 0; - ctx->buf = z_calloc(1, ctx->size + 1); + if (pipe->is_eof) { + return copy_pipe_result_finished; } - while (1) { - res = ReadFile(pipe, &ctx->buf[ctx->len], ctx->size - ctx->len, &bytes_read, NULL); + DWORD bytes_read; - if (!res && GetLastError() != ERROR_MORE_DATA) { - break; - } + if (pipe->is_pending) { + if (!GetOverlappedResult(pipe->handle, &pipe->overlapped, &bytes_read, TRUE)) { + switch (GetLastError()) { + case ERROR_BROKEN_PIPE: + pipe->is_eof = true; + if (!close_handle(&pipe->handle)) { + return copy_pipe_result_failed; + } + return copy_pipe_result_finished; + default: + win32_fatal("GetOverlappedResult:"); + return copy_pipe_result_failed; + } + } else { + if (bytes_read) { + sbuf_pushn(0, sbuf, pipe->overlapped_buf, bytes_read); + pipe->overlapped.Offset = 0; + pipe->overlapped.OffsetHigh = 0; + } - ctx->len += bytes_read; - if ((ctx->len + COPY_PIPE_BLOCK_SIZE) > ctx->size) { - ctx->size += COPY_PIPE_BLOCK_SIZE; - ctx->buf = z_realloc(ctx->buf, ctx->size + 1); - memset(ctx->buf + ctx->len, 0, (ctx->size + 1) - ctx->len); + ResetEvent(pipe->overlapped.hEvent); } } - if (!res) { - if (GetLastError() != ERROR_BROKEN_PIPE) { + if (!ReadFile(pipe->handle, + pipe->overlapped_buf, + sizeof(pipe->overlapped_buf), + &bytes_read, + &pipe->overlapped)) { + switch (GetLastError()) { + case ERROR_BROKEN_PIPE: + pipe->is_eof = true; + if (!close_handle(&pipe->handle)) { + return copy_pipe_result_failed; + } + return copy_pipe_result_finished; + case ERROR_IO_PENDING: + pipe->is_pending = true; + break; + default: + win32_fatal("ReadFile:"); return copy_pipe_result_failed; } + } else { + pipe->is_pending = false; } - /* remove remaining \r */ - buf = z_calloc(1, ctx->size + 1); - if (!buf) { - return copy_pipe_result_failed; - } - - it1 = ctx->buf; - it2 = buf; - len = 0; - for (it1 = ctx->buf; *it1; it1++) { - if (*it1 == '\r' && *(it1 + 1) == '\n') { - continue; - } - - *it2 = *it1; - it2++; - len++; - } - - z_free(ctx->buf); - ctx->buf = buf; - ctx->len = len; - - return copy_pipe_result_finished; + return copy_pipe_result_waiting; } static enum copy_pipe_result copy_pipes(struct run_cmd_ctx *ctx) { - enum copy_pipe_result res; + struct { + struct win_pipe_inst *pipe; + struct sbuf *sbuf; + } pipes[2]; + HANDLE events[2]; + uint32_t event_count = 0; - if ((res = copy_pipe(ctx->pipe_out.pipe[0], &ctx->out)) == copy_pipe_result_failed) { - return res; - } +#define PUSH_PIPE(__p, __sb) \ + if (!(__p)->is_eof) { \ + pipes[event_count].pipe = (__p); \ + pipes[event_count].sbuf = (__sb); \ + events[event_count] = (__p)->event; \ + ++event_count; \ + } \ + + PUSH_PIPE(&ctx->pipe_out, &ctx->out); + PUSH_PIPE(&ctx->pipe_err, &ctx->err); - switch (copy_pipe(ctx->pipe_err.pipe[0], &ctx->err)) { - case copy_pipe_result_waiting: +#undef PUSH_PIPE + + DWORD wait = WaitForMultipleObjects(event_count, events, FALSE, 0); + + if (wait == WAIT_TIMEOUT) { return copy_pipe_result_waiting; - case copy_pipe_result_finished: - return res; - case copy_pipe_result_failed: - return copy_pipe_result_failed; - default: - assert(false && "unreachable"); - return copy_pipe_result_failed; + } else if (wait == WAIT_FAILED) { + win32_fatal("WaitForMultipleObjects:"); + } else if (WAIT_ABANDONED_0 <= wait && wait < WAIT_ABANDONED_0 + event_count) { + win32_fatal("WaitForMultipleObjects: abandoned"); + } else if (wait >= WAIT_OBJECT_0 + event_count) { + win32_fatal("WaitForMultipleObjects: index out of range"); + } else { + wait -= WAIT_OBJECT_0; } + + return copy_pipe(pipes[wait].pipe, pipes[wait].sbuf); } static void @@ -142,10 +172,10 @@ run_cmd_ctx_close_pipes(struct run_cmd_ctx *ctx) return; } - CLOSE_PIPE(ctx->pipe_err.pipe[0]); - CLOSE_PIPE(ctx->pipe_err.pipe[1]); - CLOSE_PIPE(ctx->pipe_out.pipe[0]); - CLOSE_PIPE(ctx->pipe_out.pipe[1]); + close_handle(&ctx->pipe_err.handle); + close_handle(&ctx->pipe_err.event); + close_handle(&ctx->pipe_out.handle); + close_handle(&ctx->pipe_out.event); // TODO stdin #if 0 @@ -159,110 +189,141 @@ run_cmd_ctx_close_pipes(struct run_cmd_ctx *ctx) enum run_cmd_state run_cmd_collect(struct run_cmd_ctx *ctx) { + DWORD res; + DWORD status; + enum copy_pipe_result pipe_res = 0; - if (!(ctx->flags & run_cmd_ctx_flag_dont_capture)) { - if ((pipe_res = copy_pipes(ctx)) == copy_pipe_result_failed) { - return run_cmd_error; + bool loop = true; + while (loop) { + if (!(ctx->flags & run_cmd_ctx_flag_dont_capture)) { + if ((pipe_res = copy_pipes(ctx)) == copy_pipe_result_failed) { + return run_cmd_error; + } } - } - if (ctx->flags & run_cmd_ctx_flag_async) { - } else { - DWORD res; - DWORD status; - - res = WaitForSingleObject(ctx->process, INFINITE); + res = WaitForSingleObject(ctx->process, 1); switch (res) { + case WAIT_TIMEOUT: + if (ctx->flags & run_cmd_ctx_flag_async) { + return run_cmd_running; + } + break; case WAIT_OBJECT_0: - break; - default: - ctx->err_msg = "child exited abnormally"; - return run_cmd_error; - } - - if (!GetExitCodeProcess(ctx->process, &status)) { - ctx->err_msg = "can not get process exit code"; - return run_cmd_error; + // State is signalled + loop = false; + break; + case WAIT_FAILED: + ctx->err_msg = win32_error(); + return run_cmd_error; + case WAIT_ABANDONED: + ctx->err_msg = "child exited abnormally (WAIT_ABANDONED)"; + return run_cmd_error; } + } - ctx->status = (int)status; + if (!GetExitCodeProcess(ctx->process, &status)) { + ctx->err_msg = "can not get process exit code"; + return run_cmd_error; } + ctx->status = (int)status; + if (!(ctx->flags & run_cmd_ctx_flag_dont_capture)) { - while (pipe_res != copy_pipe_result_finished) { - if ((pipe_res = copy_pipes(ctx)) == copy_pipe_result_failed) { + while (!(ctx->pipe_out.is_eof && ctx->pipe_err.is_eof)) { + if (copy_pipes(ctx) == copy_pipe_result_failed) { return run_cmd_error; } } } - run_cmd_ctx_close_pipes(ctx); - return run_cmd_finished; } static bool -open_pipes(HANDLE *pipe, const char *name) +open_pipes(struct run_cmd_ctx *ctx, struct win_pipe_inst *pipe, const char *name) { - char buf[512]; - SECURITY_ATTRIBUTES sa; - BOOL connected; - int i; - - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = TRUE; - - i = 0; - do { - HRESULT res; - - if (FAILED(StringCchPrintf(buf, sizeof(buf), "%s%d", name, i))) { - return false; - } - pipe[0] = CreateNamedPipe(buf, PIPE_ACCESS_INBOUND, PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT, 1UL, BUF_SIZE_4k, BUF_SIZE_4k, 0UL, &sa); - i++; - } while (pipe[0] == INVALID_HANDLE_VALUE); - - if (pipe[0] == INVALID_HANDLE_VALUE) { - LOG_E("failed to create read end of the pipe: %ld %s", GetLastError(), win32_error()); + static uint64_t uniq = 0; + char pipe_name[256]; + snprintf(pipe_name, ARRAY_LEN(pipe_name), + "\\\\.\\pipe\\muon_run_cmd_pid%lu_%llu_%s", GetCurrentProcessId(), uniq, name); + ++uniq; + + if (!record_handle(&pipe->event, CreateEvent( + NULL, // default security attribute + TRUE, // manual-reset event + TRUE, // initial state = signaled + NULL // unnamed event object + ))) { + win32_fatal("CreateEvent:"); + } + + memset(&pipe->overlapped, 0, sizeof(pipe->overlapped)); + pipe->overlapped.hEvent = pipe->event; + + if (!record_handle(&pipe->handle, CreateNamedPipeA( + pipe_name, + PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_BYTE | PIPE_WAIT, + PIPE_UNLIMITED_INSTANCES, + 0, 0, INFINITE, NULL + ))) { + win32_fatal("CreateNamedPipe:"); return false; } - pipe[1] = CreateFile(buf, GENERIC_WRITE, 0UL, &sa, OPEN_EXISTING, 0UL, NULL); - if (pipe[1] == INVALID_HANDLE_VALUE) { - LOG_E("failed to create write end of the pipe: %s", win32_error()); - CLOSE_PIPE(pipe[0]); + if (!ConnectNamedPipe(pipe->handle, &pipe->overlapped) && GetLastError() != ERROR_IO_PENDING) { + win32_fatal("ConnectNamedPipe:"); return false; } - connected = ConnectNamedPipe(pipe[0], NULL); - if (connected == 0 && GetLastError() != ERROR_PIPE_CONNECTED) { - LOG_E("failed to connect to pipe: %s", win32_error()); - CLOSE_PIPE(pipe[1]); - CLOSE_PIPE(pipe[0]); - return false; + HANDLE output_write_child; + { + // Get the write end of the pipe as a handle inheritable across processes. + HANDLE output_write_handle, dup; + if (!record_handle(&output_write_handle, + CreateFileA(pipe_name, GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL))) { + win32_fatal("CreateFile:"); + return false; + } + + if (!DuplicateHandle( + GetCurrentProcess(), + output_write_handle, + GetCurrentProcess(), + &dup, + 0, + TRUE, + DUPLICATE_SAME_ACCESS)) { + win32_fatal("DuplicateHandle:"); + return false; + } + + if (!record_handle(&output_write_child, dup)) { + return false; + } else if (!close_handle(&output_write_handle)) { + return false; + } } + pipe->child_handle = output_write_child; + pipe->is_pending = true; return true; } static bool open_run_cmd_pipe(struct run_cmd_ctx *ctx) { - ctx->pipe_out.pipe[0] = INVALID_HANDLE_VALUE; - ctx->pipe_out.pipe[1] = INVALID_HANDLE_VALUE; - ctx->pipe_err.pipe[0] = INVALID_HANDLE_VALUE; - ctx->pipe_err.pipe[1] = INVALID_HANDLE_VALUE; - if (ctx->flags & run_cmd_ctx_flag_dont_capture) { return true; } - if (!open_pipes(ctx->pipe_out.pipe, "\\\\.\\pipe\\run_cmd_pipe_out")) { + sbuf_init(&ctx->out, 0, 0, sbuf_flag_overflow_alloc); + sbuf_init(&ctx->err, 0, 0, sbuf_flag_overflow_alloc); + + if (!open_pipes(ctx, &ctx->pipe_out, "out")) { return false; - } else if (!open_pipes(ctx->pipe_err.pipe, "\\\\.\\pipe\\run_cmd_pipe_err")) { + } else if (!open_pipes(ctx, &ctx->pipe_err, "err")) { return false; } @@ -274,8 +335,6 @@ open_run_cmd_pipe(struct run_cmd_ctx *ctx) static bool run_cmd_internal(struct run_cmd_ctx *ctx, char *command_line, const char *envstr, uint32_t envc) { - PROCESS_INFORMATION pi; - STARTUPINFO si; const char *p; BOOL res; @@ -343,17 +402,32 @@ run_cmd_internal(struct run_cmd_ctx *ctx, char *command_line, const char *envstr goto err; } - ZeroMemory(&si, sizeof(STARTUPINFO)); - si.cb = sizeof(STARTUPINFO); + SECURITY_ATTRIBUTES security_attributes; + memset(&security_attributes, 0, sizeof(SECURITY_ATTRIBUTES)); + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = TRUE; + + // Must be inheritable so subprocesses can dup to children. + // TODO: delete when stdin support added + HANDLE nul; + if (!record_handle(&nul, CreateFileA("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL))) { + error_unrecoverable("couldn't open nul"); + } + + STARTUPINFOA startup_info; + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(STARTUPINFO); if (!(ctx->flags & run_cmd_ctx_flag_dont_capture)) { - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdOutput = ctx->pipe_out.pipe[1]; - // FIXME stdin - si.hStdInput = NULL; - si.hStdError = ctx->pipe_err.pipe[1]; + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = nul; + startup_info.hStdOutput = ctx->pipe_out.child_handle; + startup_info.hStdError = ctx->pipe_err.child_handle; } - ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); + PROCESS_INFORMATION process_info; + memset(&process_info, 0, sizeof(process_info)); if (ctx->chdir) { if (!fs_dir_exists(ctx->chdir)) { @@ -362,17 +436,26 @@ run_cmd_internal(struct run_cmd_ctx *ctx, char *command_line, const char *envstr } } - res = CreateProcess(NULL, command_line, NULL, NULL, TRUE, 0UL, NULL, ctx->chdir, &si, &pi); + DWORD process_flags = 0; + + res = CreateProcess(NULL, command_line, NULL, NULL, + /* inherit handles */ TRUE, process_flags, + NULL, ctx->chdir, + &startup_info, &process_info); if (!res) { + DWORD error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND) { + } LOG_E("CreateProcess() failed: %s", win32_error()); goto err; } - CLOSE_PIPE(ctx->pipe_out.pipe[1]); - CLOSE_PIPE(ctx->pipe_err.pipe[1]); + close_handle(&ctx->pipe_out.child_handle); + close_handle(&ctx->pipe_err.child_handle); + close_handle(&nul); - ctx->process = pi.hProcess; - CloseHandle(pi.hThread); + record_handle(&ctx->process, process_info.hProcess); + CloseHandle(process_info.hThread); if (ctx->flags & run_cmd_ctx_flag_async) { return true; @@ -433,14 +516,14 @@ argv_to_command_line(struct run_cmd_ctx *ctx, struct source *src, const char *ar sbuf_push(NULL, cmd, '\"'); sbuf_pushs(NULL, cmd, cmd_argv0->buf); sbuf_push(NULL, cmd, '\"'); - }else if (fs_has_extension(cmd_argv0->buf, ".bat")) { + } else if (fs_has_extension(cmd_argv0->buf, ".bat")) { /* * to run .bat file, run it with cmd.exe /c */ sbuf_pushs(NULL, cmd, "\"c:\\windows\\system32\\cmd.exe\" \"/c\" \""); sbuf_pushs(NULL, cmd, cmd_argv0->buf); sbuf_push(NULL, cmd, '\"'); - }else { + } else if (fs_exists(cmd_argv0->buf)) { if (!fs_read_entire_file(cmd_argv0->buf, src)) { ctx->err_msg = "error determining command interpreter"; return false; @@ -488,7 +571,6 @@ argv_to_command_line(struct run_cmd_ctx *ctx, struct source *src, const char *ar } } } else { - if (!fs_find_cmd(NULL, cmd_argv0, argv0)) { ctx->err_msg = "command not found"; return false; @@ -547,6 +629,12 @@ argv_to_command_line(struct run_cmd_ctx *ctx, struct source *src, const char *ar return true; } +bool +run_cmd_unsplit(struct run_cmd_ctx *ctx, char *cmd, const char *envstr, uint32_t envc) +{ + return run_cmd_internal(ctx, cmd, envstr, envc); +} + bool run_cmd_argv(struct run_cmd_ctx *ctx, char *const *argtab, const char *envstr, uint32_t envc) { @@ -594,18 +682,11 @@ run_cmd(struct run_cmd_ctx *ctx, const char *argstr, uint32_t argc, const char * void run_cmd_ctx_destroy(struct run_cmd_ctx *ctx) { - CloseHandle(ctx->process); + close_handle(&ctx->process); run_cmd_ctx_close_pipes(ctx); - if (ctx->out.size) { - z_free(ctx->out.buf); - ctx->out.size = 0; - } - - if (ctx->err.size) { - z_free(ctx->err.buf); - ctx->err.size = 0; - } + sbuf_destroy(&ctx->out); + sbuf_destroy(&ctx->err); } bool From fd23afc960876aa81610e7afed45afcffd1cdb62 Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Thu, 25 Jan 2024 06:16:02 -0500 Subject: [PATCH 51/52] use sbufs in posix run cmd --- src/platform/posix/run_cmd.c | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/platform/posix/run_cmd.c b/src/platform/posix/run_cmd.c index 0831e06b3..2309970c6 100644 --- a/src/platform/posix/run_cmd.c +++ b/src/platform/posix/run_cmd.c @@ -17,6 +17,7 @@ #include "args.h" #include "buf_size.h" +#include "error.h" #include "log.h" #include "platform/filesystem.h" #include "platform/mem.h" @@ -25,8 +26,6 @@ extern char **environ; -#define COPY_PIPE_BLOCK_SIZE BUF_SIZE_1k - enum copy_pipe_result { copy_pipe_result_finished, copy_pipe_result_waiting, @@ -34,17 +33,13 @@ enum copy_pipe_result { }; static enum copy_pipe_result -copy_pipe(int pipe, struct run_cmd_pipe_ctx *ctx) +copy_pipe(int pipe, struct sbuf *sbuf) { ssize_t b; - if (!ctx->size) { - ctx->size = COPY_PIPE_BLOCK_SIZE; - ctx->len = 0; - ctx->buf = z_calloc(1, ctx->size + 1); - } + char buf[4096]; while (true) { - b = read(pipe, &ctx->buf[ctx->len], ctx->size - ctx->len); + b = read(pipe, buf, sizeof(buf)); if (b == -1) { if (errno == EAGAIN) { @@ -56,12 +51,7 @@ copy_pipe(int pipe, struct run_cmd_pipe_ctx *ctx) return copy_pipe_result_finished; } - ctx->len += b; - if ((ctx->len + COPY_PIPE_BLOCK_SIZE) > ctx->size) { - ctx->size *= 2; - ctx->buf = z_realloc(ctx->buf, ctx->size + 1); - memset(&ctx->buf[ctx->len], 0, (ctx->size + 1) - ctx->len); - } + sbuf_pushn(0, sbuf, buf, b); } } @@ -253,6 +243,9 @@ run_cmd_internal(struct run_cmd_ctx *ctx, const char *_cmd, char *const *argv, c } if (!(ctx->flags & run_cmd_ctx_flag_dont_capture)) { + sbuf_init(&ctx->out, 0, 0, sbuf_flag_overflow_alloc); + sbuf_init(&ctx->err, 0, 0, sbuf_flag_overflow_alloc); + if (!open_run_cmd_pipe(ctx->pipefd_out, ctx->pipefd_out_open)) { goto err; } else if (!open_run_cmd_pipe(ctx->pipefd_err, ctx->pipefd_err_open)) { @@ -496,15 +489,8 @@ run_cmd_ctx_destroy(struct run_cmd_ctx *ctx) { run_cmd_ctx_close_fds(ctx); - if (ctx->out.size) { - z_free(ctx->out.buf); - ctx->out.size = 0; - } - - if (ctx->err.size) { - z_free(ctx->err.buf); - ctx->err.size = 0; - } + sbuf_destroy(&ctx->out); + sbuf_destroy(&ctx->err); } bool From 794d2721b2889ea43fbd0f08c2a9bf1227e9c58b Mon Sep 17 00:00:00 2001 From: Stone Tickle Date: Thu, 25 Jan 2024 06:16:14 -0500 Subject: [PATCH 52/52] fix warnings --- src/external/samurai/deps.c | 10 +++++----- src/external/samurai/util.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/external/samurai/deps.c b/src/external/samurai/deps.c index e243462a7..c34591ba3 100644 --- a/src/external/samurai/deps.c +++ b/src/external/samurai/deps.c @@ -153,7 +153,7 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) { char *depspath = (char *)ninja_depsname; uint32_t *buf, cap, ver, sz, id; - size_t len, i, j, nrecord; + size_t len, i, j; bool isdep; struct samu_string *path; struct samu_node *n; @@ -199,7 +199,7 @@ samu_depsinit(struct samu_ctx *ctx, const char *builddir) samu_warn("unknown deps log version"); goto rewrite; } - for (nrecord = 0;; ++nrecord) { + while (true) { if (src_fread(&sz, sizeof(sz), 1, &src) != 1) { break; } @@ -389,7 +389,7 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) case '$': c = src_getc(&src); if (c != '$') { - samu_warn("bad depfile[%lld]: contains variable reference", src.i); + samu_warn("bad depfile[%d]: contains variable reference", (int)src.i); goto err; } break; @@ -399,7 +399,7 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) } if (sawcolon) { if (!isspace(c) && c != EOF) { - samu_warn("bad depfile[%lld]: '%c' is not a valid target character", src.i, c); + samu_warn("bad depfile[%d]: '%c' is not a valid target character", (int)src.i, c); goto err; } if (ctx->deps.buf.len > 0) { @@ -448,7 +448,7 @@ samu_depsparse(struct samu_ctx *ctx, const char *name, bool allowmissing) for (;;) { if (c == '\\') { if (src_getc(&src) != '\n') { - samu_warn("bad depfile[%lld]: '\\' only allowed before newline", src.i); + samu_warn("bad depfile[%d]: '\\' only allowed before newline", (int)src.i); printf("%s", src.src.src); goto err; } diff --git a/src/external/samurai/util.c b/src/external/samurai/util.c index e4e643d7d..933949bc4 100644 --- a/src/external/samurai/util.c +++ b/src/external/samurai/util.c @@ -120,7 +120,7 @@ samu_arena_destroy(struct samu_arena *a) z_free(a->blocks[i]); } - L("samu allocd %d blocks, a:%d, f:%d, r:%3.3f\n", a->blocks_len, a->allocd, a->filled, (float)a->filled / (float)a->allocd * 100.0f); + L("samu allocd %d blocks, a:%d, f:%d, r:%3.3f\n", (int)a->blocks_len, (int)a->allocd, (int)a->filled, (float)a->filled / (float)a->allocd * 100.0f); z_free(a->blocks); }