141231.1 · ae7f8d6ab9d29bd72f462fdafa4d7a9270706d3b
Further module system revamp (fix #659)
To import a module now use:
# Import module.jq file:
import "relative/path/to/module" as foo;
# Use the module's defs as foo::<def-name>
To import a JSON file:
# Read file.json:
import "relative/path/to/file" as $foo;
#
# Use as $foo::foo
Using `-L` now drops the builtin library path and appends the requested path to the empty array (or the result of an earlier `-L`).
Support for the `$JQ_LIBRARY_PATH` environment variable has been removed.
reading declarations
builtin.c
@@ -824,6 +824,21 @@ static jv f_env(jq_state *jq, jv input) { return env; } +static jv f_get_search_list(jq_state *jq, jv input) {+ jv_free(input);+ return jq_get_lib_dirs(jq);+}++static jv f_get_prog_origin(jq_state *jq, jv input) {+ jv_free(input);+ return jq_get_prog_origin(jq);+}++static jv f_get_jq_origin(jq_state *jq, jv input) {+ jv_free(input);+ return jq_get_jq_origin(jq);+}+ static jv f_string_split(jq_state *jq, jv a, jv b) { if (jv_get_kind(a) != JV_KIND_STRING || jv_get_kind(b) != JV_KIND_STRING) { jv_free(a);@@ -937,6 +952,9 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_error, "error", 2}, {(cfunction_ptr)f_format, "format", 2}, {(cfunction_ptr)f_env, "env", 1},+ {(cfunction_ptr)f_get_search_list, "get_search_list", 1},+ {(cfunction_ptr)f_get_prog_origin, "get_prog_origin", 1},+ {(cfunction_ptr)f_get_jq_origin, "get_jq_origin", 1}, {(cfunction_ptr)f_match, "_match_impl", 4}, {(cfunction_ptr)f_modulemeta, "modulemeta", 1}, {(cfunction_ptr)f_input, "_input", 1},
bytecode.c
@@ -9,6 +9,7 @@ #define NONE 0, 1 #define CONSTANT OP_HAS_CONSTANT, 2 #define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3+#define GLOBAL (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4 #define BRANCH OP_HAS_BRANCH, 2 #define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3 #define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
compile.c
@@ -148,6 +148,15 @@ block gen_const(jv constant) { return inst_block(i); } +block gen_const_global(jv constant, const char *name) {+ assert((opcode_describe(STORE_GLOBAL)->flags & (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING)) ==+ (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING));+ inst* i = inst_new(STORE_GLOBAL);+ i->imm.constant = constant;+ i->symbol = strdup(name);+ return inst_block(i);+}+ int block_is_const(block b) { return (block_is_single(b) && b.first->op == LOADK); }@@ -239,6 +248,10 @@ int block_has_only_binders_and_imports(block binders, int bindflags) { return 1; } +static int inst_is_binder(inst *i, int bindflags) {+ return !((opcode_describe(i->op)->flags & bindflags) != bindflags && i->op != MODULEMETA);+}+ int block_has_only_binders(block binders, int bindflags) { bindflags |= OP_HAS_BINDING; bindflags &= ~OP_BIND_WILDCARD;@@ -311,10 +324,6 @@ static int block_bind_subblock(block binder, block body, int bindflags, int brea ((bindflags & OP_BIND_WILDCARD) && i->symbol[0] == '*' && break_distance <= 3 && (i->symbol[1] == '1' + break_distance) && i->symbol[2] == '\0'))) {- if (bindflags & OP_BIND_WILDCARD) {- jv_mem_free(i->symbol);- i->symbol = strdup(binder.first->symbol);- } // bind this instruction if (i->op == CALL_JQ && i->nactuals == -1) i->nactuals = block_count_actuals(i->arglist);@@ -353,7 +362,6 @@ block block_bind(block binder, block body, int bindflags) { } block block_bind_library(block binder, block body, int bindflags, const char* libname) {- assert(block_has_only_binders(binder, bindflags)); bindflags |= OP_HAS_BINDING; int nrefs = 0; int matchlen = strlen(libname);@@ -363,13 +371,21 @@ block block_bind_library(block binder, block body, int bindflags, const char* li strcpy(matchname+matchlen,"::"); matchlen += 2; }+ assert(block_has_only_binders(binder, bindflags)); for (inst *curr = binder.first; curr; curr = curr->next) {+ int bindflags2 = bindflags; char* cname = curr->symbol; char* tname = malloc(strlen(curr->symbol)+matchlen+1); strcpy(tname, matchname); strcpy(tname+matchlen,cname);++ // Ew+ if ((opcode_describe(curr->op)->flags & (OP_HAS_VARIABLE | OP_HAS_CONSTANT)))+ bindflags2 = OP_HAS_VARIABLE | OP_HAS_BINDING;++ // This mutation is ugly, even if we undo it curr->symbol = tname;- nrefs += block_bind_subblock(inst_block(curr), body, bindflags, 0);+ nrefs += block_bind_subblock(inst_block(curr), body, bindflags2, 0); curr->symbol = cname; free(tname); }@@ -440,15 +456,13 @@ jv block_take_imports(block* body) { jv imports = jv_array(); inst* top = NULL;- if (body->first->op == TOP) {+ if (body->first && body->first->op == TOP) { top = block_take(body); } while (body->first && (body->first->op == MODULEMETA || body->first->op == DEPS)) { inst* dep = block_take(body); if (dep->op == DEPS) {- jv opts = jv_copy(dep->imm.constant);- opts = jv_object_set(opts,jv_string("name"),jv_string(dep->symbol));- imports = jv_array_append(imports, opts);+ imports = jv_array_append(imports, jv_copy(dep->imm.constant)); } inst_free(dep); }@@ -458,13 +472,11 @@ jv block_take_imports(block* body) { return imports; } -block gen_module(const char* name, block metadata) {+block gen_module(block metadata) { inst* i = inst_new(MODULEMETA);- i->symbol = strdup(name); i->imm.constant = block_const(metadata); if (jv_get_kind(i->imm.constant) != JV_KIND_OBJECT) i->imm.constant = jv_object_set(jv_object(), jv_string("metadata"), i->imm.constant);- i->imm.constant = jv_object_set(i->imm.constant, jv_string("name"), jv_string(name)); block_free(metadata); return inst_block(i); }@@ -475,17 +487,17 @@ jv block_module_meta(block b) { return jv_null(); } -block gen_import(const char* name, block metadata, const char* as) {+block gen_import(const char* name, block metadata, const char* as, int is_data) { assert(metadata.first == NULL || block_is_const(metadata)); inst* i = inst_new(DEPS);- i->symbol = strdup(name); jv meta; if (block_is_const(metadata)) meta = block_const(metadata); else meta = jv_object();- if (as)- meta = jv_object_set(meta, jv_string("as"), jv_string(as));+ meta = jv_object_set(meta, jv_string("as"), jv_string(as));+ meta = jv_object_set(meta, jv_string("is_data"), is_data ? jv_true() : jv_false());+ meta = jv_object_set(meta, jv_string("relpath"), jv_string(name)); i->imm.constant = meta; block_free(metadata); return inst_block(i);@@ -797,7 +809,11 @@ block gen_or(block a, block b) { } block gen_var_binding(block var, const char* name, block body) {- return BLOCK(gen_op_simple(DUP), var,+ // var bindings can be added after coding the program; leave the TOP first.+ block top = gen_noop();+ if (body.first && body.first->op == TOP)+ top = inst_block(block_take(&body));+ return BLOCK(top, gen_op_simple(DUP), var, block_bind(gen_op_unbound(STOREV, name), body, OP_HAS_VARIABLE)); }@@ -1084,6 +1100,13 @@ static int compile(struct bytecode* bc, block b) { code[pos++] = nesting_level(bc, arg->bound_by); code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE; }+ } else if ((op->flags & OP_HAS_CONSTANT) && (op->flags & OP_HAS_VARIABLE)) {+ // STORE_GLOBAL: constant global, basically+ code[pos++] = jv_array_length(jv_copy(constant_pool));+ constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));+ code[pos++] = nesting_level(bc, curr->bound_by);+ uint16_t var = (uint16_t)curr->bound_by->imm.intval;+ code[pos++] = var; } else if (op->flags & OP_HAS_CONSTANT) { code[pos++] = jv_array_length(jv_copy(constant_pool)); constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
compile.h
@@ -20,6 +20,7 @@ block gen_noop(); int block_is_noop(block b); block gen_op_simple(opcode op); block gen_const(jv constant);+block gen_const_global(jv constant, const char *name); int block_is_const(block b); jv_kind block_const_kind(block b); jv block_const(block b);@@ -28,9 +29,9 @@ block gen_op_unbound(opcode op, const char* name); block gen_op_bound(opcode op, block binder); block gen_op_var_fresh(opcode op, const char* name); -block gen_module(const char* name, block metadata);+block gen_module(block metadata); jv block_module_meta(block b);-block gen_import(const char* name, block metadata, const char *as);+block gen_import(const char* name, block metadata, const char *as, int is_data); block gen_function(const char* name, block formals, block body); block gen_param_regular(const char* name); block gen_param(const char* name);
docs/content/3.manual/manual.yml
@@ -185,8 +185,9 @@ sections: * `-Ldirectory` / `-L directory`: - Add `directory` to the search list for modules. See the section- on modules below.+ Prepend `directory` to the search list for modules. If this+ option is used then no builtin search list is used. See the+ section on modules below. * `-e` / `--exit-status`: @@ -2474,9 +2475,7 @@ sections: body: | jq has a library/module system. Modules are files whose names end- in `.jq`. Modules should (but don't have to) start with a- `module` directive. Programs and modules must `import` their- dependencies, if any.+ in `.jq`. Modules imported by a program are searched for in a default search path (see below). The `import` directive allows the importer to@@ -2498,29 +2497,20 @@ sections: the default is appended. The default search path is the search path given to the `-L`- command-line option, else the "$JQ_LIBRARY_PATH", if set in the- environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or- `["~/.jq", "$ORIGIN/lib"]` (on Windows).-- The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's- string values are split on ":" on Unix, or ";" on Windows.+ command-line option, else `["~/.jq", "$ORIGIN/../lib/jq",+ "$ORIGIN/../lib"]`. Null and empty string path elements terminate search path processing. - Modules may be organized into a hierarchy, with name components- separated by double-colons ("::"). For example: "foo::bar". Each- component in the name corresponds to a directory on the- filesystem.-- A module named "foo::bar" would be searched for in "foo/bar.jq"- and "foo/bar/bar.jq" in the given search path. This is intended to- allow modules to be placed in a directory along with, for example,- version control files, README files, and so on, but also to allow- for single-file modules.+ A dependency with relative path "foo/bar" would be searched for in+ "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This+ is intended to allow modules to be placed in a directory along+ with, for example, version control files, README files, and so on,+ but also to allow for single-file modules. Consecutive components with the same name are not allowed to avoid- ambiguities (e.g., "foo::foo").+ ambiguities (e.g., "foo/foo"). For example, with `-L$HOME/.jq` a module `foo` can be found in `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`.@@ -2528,38 +2518,78 @@ sections: If "$HOME/.jq" is a file, it is sourced into the main program. entries:- - title: "`module NAME {<metadata>};`"+ - title: "`import RelativePathString as NAME;`"+ body: |++ Imports a module found at the given path relative to a+ directory in a search path. The module's symbols are prefixed+ with "NAME::". A ".jq" suffix will be added to the relative+ path string.++ - title: "`import RelativePathString as NAME {<metadata>};`" body: | - Defines a module named `NAME` with the given metadata. If- given it must be the first thing in the module. Programs also- may use this directive.+ Imports a module found at the given path relative to a+ directory in a search path. The module's symbols are prefixed+ with "NAME::". A ".jq" suffix will be added to the relative+ path string. The metadata must be a constant jq expression. It should be- an object with keys like "version", "repo", "URI", and so on.- At this time jq doesn't use this metadata, but it is made- available to users via the `modulemeta` builtin.+ an object with keys like "homepage" and so on. At this time+ jq only uses the "search" key/value of the metadate. The+ metadata is also made available to users via the `modulemeta`+ builtin.+ + The "search" key in the metadata, if present, should have a+ string or array value (array of strings); this is the search+ path to be prefixed to the top-level search path. - - title: "`import NAME {<metadata>};` `import NAME {<metadata>} as NAME;`"+ - title: "`import RelativePathString as NAME;`" body: | - Imports a module named `NAME`. If the second form is used- then the module's symbols are prefixed with "NAME::".+ Imports a JSON file found at the given path relative to a+ directory in a search path. The file's data will be available+ as `$NAME::NAME`. A ".json" suffix will be added to the+ relative path string.++ - title: "`import RelativePathString as $NAME {<metadata>};`"+ body: |++ Imports a JSON file found at the given path relative to a+ directory in a search path. The file's data will be available+ as `$NAME::NAME`. A ".json" suffix will be added to the+ relative path string. The metadata must be a constant jq expression. It should be- an object with keys like "version", "repo", "URI", and so on.+ an object with keys like "homepage" and so on. At this time+ jq only uses the "search" key/value of the metadate. The+ metadata is also made available to users via the `modulemeta`+ builtin. The "search" key in the metadata, if present, should have a string or array value (array of strings); this is the search path to be prefixed to the top-level search path. + - title: "`module {<metadata>};`"+ body: |++ This directive is entirely optional. It's not required for+ proper operation. It serves only the purpose of providing+ metadata that can be read with the `modulemeta` builtin.++ The metadata must be a constant jq expression. It should be+ an object with keys like "homepage". At this time jq doesn't+ use this metadata, but it is made available to users via the+ `modulemeta` builtin.+ - title: "`modulemeta`" body: | - Takes a module name as input and outputs the module's- metadata, with the module's imports (including metadata) as- the "deps" key.+ Takes a module name as input and outputs the module's metadata+ as an object, with the module's imports (including metadata)+ as an array value for the "deps" key. Programs can use this to query a module's metadata, which they could then use to, for example, search for, download, and install missing dependencies.+
execute.c
@@ -508,6 +508,25 @@ jv jq_next(jq_state *jq) { break; } + case STORE_GLOBAL: {+ // Get the constant+ jv val = jv_array_get(jv_copy(frame_current(jq)->bc->constants), *pc++);+ assert(jv_is_valid(val));++ // Store the var+ uint16_t level = *pc++;+ uint16_t v = *pc++;+ jv* var = frame_local_var(jq, v, level);+ if (jq->debug_trace_enabled) {+ printf("V%d = ", v);+ jv_dump(jv_copy(val), 0);+ printf(" (%d)\n", jv_get_refcnt(val));+ }+ jv_free(*var);+ *var = val;+ break;+ }+ case PATH_BEGIN: { jv v = stack_pop(jq); stack_push(jq, jq->path);
linker.c
@@ -21,33 +21,71 @@ struct lib_loading_state { block *defs; uint64_t ct; };-static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_loading_state *lib_state);+static int load_library(jq_state *jq, jv lib_path, int is_data, int raw,+ const char *as, block *out_block,+ struct lib_loading_state *lib_state);+ // Given a lib_path to search first, creates a chain of search paths // in the following order: // 1. lib_path-// 2. -L paths passed in on the command line (from jq_state*)-// 3. JQ_LIBRARY_PATH environment variable-jv build_lib_search_chain(jq_state *jq, jv search_path) {+// 2. -L paths passed in on the command line (from jq_state*) or builtin list+static jv build_lib_search_chain(jq_state *jq, jv search_path, jv jq_origin, jv lib_origin) { assert(jv_get_kind(search_path) == JV_KIND_ARRAY); jv expanded = jv_array();+ jv expanded_elt; jv_array_foreach(search_path, i, path) {- path = expand_path(path);- if (jv_is_valid(path))- expanded = jv_array_append(expanded, expand_path(path));- else- jv_free(path);+ if (strcmp(".",jv_string_value(path)) == 0) {+ expanded_elt = jv_copy(path);+ } else if (jv_get_kind(lib_origin) == JV_KIND_STRING &&+ strncmp("./", jv_string_value(path),sizeof("./")-1) == 0) {+ expanded_elt = jv_string_fmt("%s/%s",+ jv_string_value(lib_origin),+ jv_string_value(path) + sizeof ("./") - 1);+ } else if (strncmp("$ORIGIN/",jv_string_value(path),sizeof("$ORIGIN/")-1) == 0) {+ expanded_elt = jv_string_fmt("%s/%s",+ jv_string_value(jq_origin),+ jv_string_value(path) + sizeof ("$ORIGIN/") - 1);+ } else {+ expanded_elt = expand_path(path);+ if (!jv_is_valid(expanded_elt)) {+ jv_free(search_path);+ jv_free(expanded);+ jv_free(path);+ return expanded_elt;+ }+ path = jv_invalid();+ }+ expanded = jv_array_append(expanded, expanded_elt);+ jv_free(path); }+ jv_free(jq_origin);+ jv_free(lib_origin); jv_free(search_path);- expanded = jv_array_concat(expanded, jq_get_lib_dirs(jq)); return expanded; } -static jv name2relpath(jv name) {- jv components = jv_string_split(jv_copy(name), jv_string("::"));+// Doesn't actually check that name not be an absolute path; we could+// just always prepend "./"!+static jv validate_relpath(jv name) {+ const char *s = jv_string_value(name);+ if (strchr(s, '\\')) {+ jv res = jv_invalid_with_msg(jv_string_fmt("Modules must be named by relative paths using '/', not '\\' (%s)", s));+ jv_free(name);+ return res;+ }+ jv components = jv_string_split(jv_copy(name), jv_string("/")); jv rp = jv_array_get(jv_copy(components), 0); components = jv_array_slice(components, 1, jv_array_length(jv_copy(components))); jv_array_foreach(components, i, x) {+ if (!strcmp(jv_string_value(x), "..")) {+ jv_free(x);+ jv_free(rp);+ jv_free(components);+ jv res = jv_invalid_with_msg(jv_string_fmt("Relative paths to modules may not traverse to parent directories (%s)", s));+ jv_free(name);+ return res;+ } if (i > 0 && jv_equal(jv_copy(x), jv_array_get(jv_copy(components), i - 1))) { jv_free(x); jv_free(rp);@@ -64,22 +102,33 @@ static jv name2relpath(jv name) { return rp; } -static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) {- assert(jv_get_kind(lib_search_path) == JV_KIND_ARRAY);- assert(jv_get_kind(lib_name) == JV_KIND_STRING);+// Assumes name has been validated+static jv jv_basename(jv name) {+ const char *s = jv_string_value(name);+ const char *p = strrchr(s, '/');+ if (!p)+ return name;+ jv res = jv_string_fmt("%s", p);+ jv_free(name);+ return res;+} - jv rel_path = name2relpath(jv_copy(lib_name));- if (!jv_is_valid(rel_path)) {- jv_free(lib_name);- return rel_path;- }+// Asummes validated relative path to module+static jv find_lib(jq_state *jq, jv rel_path, jv search, const char *suffix, jv jq_origin, jv lib_origin) {+ if (jv_get_kind(search) != JV_KIND_ARRAY)+ return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array"));+ if (jv_get_kind(rel_path) != JV_KIND_STRING)+ return jv_invalid_with_msg(jv_string_fmt("Module path must be a string")); struct stat st; int ret; - jv lib_search_paths = build_lib_search_chain(jq, lib_search_path);+ // Ideally we should cache this somewhere+ search = build_lib_search_chain(jq, search, jq_origin, lib_origin); - jv_array_foreach(lib_search_paths, i, spath) {+ jv bname = jv_basename(jv_copy(rel_path));++ jv_array_foreach(search, i, spath) { if (jv_get_kind(spath) == JV_KIND_NULL) { jv_free(spath); break;@@ -89,43 +138,44 @@ static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { jv_free(spath); continue; /* XXX report non-strings in search path?? */ }- jv testpath = jq_realpath(jv_string_fmt("%s/%s.jq",+ // Try .../module/last/component.jq+ jv testpath = jq_realpath(jv_string_fmt("%s/%s%s", jv_string_value(spath),- jv_string_value(rel_path)));+ jv_string_value(rel_path),+ suffix)); ret = stat(jv_string_value(testpath),&st); if (ret == -1 && errno == ENOENT) { jv_free(testpath);- testpath = jq_realpath(jv_string_fmt("%s/%s/%s.jq",+ // Try .../module/last/component/component.jq+ testpath = jq_realpath(jv_string_fmt("%s/%s/%s%s", jv_string_value(spath), jv_string_value(rel_path),- jv_string_value(lib_name)));+ jv_string_value(bname),+ suffix)); ret = stat(jv_string_value(testpath),&st); } if (ret == 0) {- jv_free(spath); jv_free(rel_path);- jv_free(lib_name);- jv_free(lib_search_paths);+ jv_free(search);+ jv_free(bname);+ jv_free(spath); return testpath; } jv_free(testpath); jv_free(spath); }- jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(lib_name)));+ jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(rel_path))); jv_free(rel_path);- jv_free(lib_name);- jv_free(lib_search_paths);+ jv_free(search);+ jv_free(bname); return output; } -static int version_matches(jq_state *jq, block importer, block module) {- return 1;-}--static jv default_search(jv value) {+static jv default_search(jq_state *jq, jv value) { if (!jv_is_valid(value)) {+ // dependent didn't say; prepend . to system search path listj jv_free(value);- return JV_ARRAY(jv_string("."), jv_string("$ORIGIN"));+ return jv_array_concat(JV_ARRAY(jv_string(".")), jq_get_lib_dirs(jq)); } if (jv_get_kind(value) != JV_KIND_ARRAY) return JV_ARRAY(value);@@ -139,38 +189,23 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block int nerrors = 0; jv_array_foreach(deps, i, dep) {- jv name = jv_object_get(jv_copy(dep), jv_string("name"));+ int is_data = jv_get_kind(jv_object_get(jv_copy(dep), jv_string("is_data"))) == JV_KIND_TRUE;+ int raw = 0;+ jv v = jv_object_get(jv_copy(dep), jv_string("raw"));+ if (jv_get_kind(v) == JV_KIND_TRUE)+ raw = 1;+ jv_free(v);+ jv relpath = validate_relpath(jv_object_get(jv_copy(dep), jv_string("relpath"))); jv as = jv_object_get(jv_copy(dep), jv_string("as"));- if (!jv_is_valid(as)) {- jv_free(as);- as = jv_string("");- }- jv search = default_search(jv_object_get(dep, jv_string("search")));- jv_array_foreach(search, k, search_elt) {- if (strcmp(".",jv_string_value(search_elt)) == 0) {- jv tsearch = jv_copy(lib_origin);- jv_free(search_elt);- search = jv_array_set(search, k, tsearch);- } else if (strncmp("./",jv_string_value(search_elt),sizeof("./")-1) == 0) {- jv tsearch = jv_string_fmt("%s/%s",- jv_string_value(lib_origin),- jv_string_value(search_elt) + sizeof ("./") - 1);- jv_free(search_elt);- search = jv_array_set(search, k, tsearch);- } else if (strncmp("$ORIGIN/",jv_string_value(search_elt),sizeof("$ORIGIN/")-1) == 0) {- jv tsearch = jv_string_fmt("%s/%s",- jv_string_value(jq_origin),- jv_string_value(search_elt) + sizeof ("$ORIGIN/") - 1);- jv_free(search_elt);- search = jv_array_set(search, k, tsearch);- } else {- jv_free(search_elt);- }- }- jv lib_path = find_lib(jq, name, search);+ assert(jv_is_valid(as) && jv_get_kind(as) == JV_KIND_STRING);+ jv search = default_search(jq, jv_object_get(dep, jv_string("search")));+ // dep is now freed; do not reuse++ // find_lib does a lot of work that could be cached...+ jv resolved = find_lib(jq, relpath, search, is_data ? ".json" : ".jq", jv_copy(jq_origin), jv_copy(lib_origin)); // XXX ...move the rest of this into a callback.- if (!jv_is_valid(lib_path)) {- jv emsg = jv_invalid_get_msg(lib_path);+ if (!jv_is_valid(resolved)) {+ jv emsg = jv_invalid_get_msg(resolved); jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg))); jv_free(emsg); jv_free(as);@@ -181,26 +216,20 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block } uint64_t state_idx = 0; for (; state_idx < lib_state->ct; ++state_idx) {- if (strcmp(lib_state->names[state_idx],jv_string_value(lib_path)) == 0)+ if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0) break; } if (state_idx < lib_state->ct) { // Found- // XXX Check version matching here!- if (version_matches(jq, bk, lib_state->defs[state_idx]))- bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as));- else- // XXX Would be nice to have the dependent's name here too- jq_report_error(jq, jv_string_fmt("jq: error: version mismatch for %s", jv_string_value(name)));- jv_free(lib_path);+ jv_free(resolved);+ // Bind the library to the program+ bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as)); } else { // Not found. Add it to the table before binding. block dep_def_block = gen_noop();- nerrors += load_library(jq, lib_path, &dep_def_block, lib_state);+ nerrors += load_library(jq, resolved, is_data, raw, jv_string_value(as), &dep_def_block, lib_state);+ // resolved has been freed if (nerrors == 0) {- // XXX Check version matching here!- if (version_matches(jq, bk, dep_def_block))- bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as));- else- jq_report_error(jq, jv_string_fmt("jq: error: version mismatch for %s", jv_string_value(name)));+ // Bind the library to the program+ bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as)); } } jv_free(as);@@ -213,30 +242,48 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block // Loads the library at lib_path into lib_state, putting the library's defs // into *out_block-static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_loading_state *lib_state) {+static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, const char *as, block *out_block, struct lib_loading_state *lib_state) { int nerrors = 0;- struct locfile* src;+ struct locfile* src = NULL; block program;- jv data = jv_load_file(jv_string_value(lib_path), 1);+ jv data;+ if (is_data && !raw)+ data = jv_load_file(jv_string_value(lib_path), 0);+ else+ data = jv_load_file(jv_string_value(lib_path), 1); int state_idx;- if (jv_is_valid(data)) {+ if (!jv_is_valid(data)) {+ if (jv_invalid_has_msg(jv_copy(data)))+ data = jv_invalid_get_msg(data);+ else+ data = jv_string("unknown error");+ jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n", jv_string_value(lib_path), jv_string_value(data)));+ nerrors++;+ goto out;+ } else if (is_data) {+ // import "foo" as $bar;+ program = gen_const_global(jv_copy(data), as);+ } else {+ // import "foo" as bar; src = locfile_init(jq, jv_string_value(data), jv_string_length_bytes(jv_copy(data))); nerrors += jq_parse_library(src, &program); if (nerrors == 0) {- state_idx = lib_state->ct++;- lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *));- lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block));- lib_state->names[state_idx] = strdup(jv_string_value(lib_path));- lib_state->defs[state_idx] = program; char *lib_origin = strdup(jv_string_value(lib_path)); nerrors += process_dependencies(jq, jq_get_jq_origin(jq), jv_string(dirname(lib_origin)),- &lib_state->defs[state_idx], lib_state);+ &program, lib_state); free(lib_origin);- *out_block = lib_state->defs[state_idx]; }- locfile_free(src); }+ state_idx = lib_state->ct++;+ lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *));+ lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block));+ lib_state->names[state_idx] = strdup(jv_string_value(lib_path));+ lib_state->defs[state_idx] = program;+ *out_block = program;+ if (src)+ locfile_free(src);+out: jv_free(lib_path); jv_free(data); return nerrors;@@ -244,8 +291,11 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ // FIXME It'd be nice to have an option to search the same search path // as we do in process_dependencies.-jv load_module_meta(jq_state *jq, jv modname) {- jv lib_path = find_lib(jq, modname, jv_array());+jv load_module_meta(jq_state *jq, jv mod_relpath) {+ // We can't know the caller's origin; we could though, if it was passed in+ jv lib_path = find_lib(jq, validate_relpath(mod_relpath), jq_get_lib_dirs(jq), ".jq", jq_get_jq_origin(jq), jv_null());+ if (!jv_is_valid(lib_path))+ return lib_path; jv meta = jv_null(); jv data = jv_load_file(jv_string_value(lib_path), 1); if (jv_is_valid(data)) {@@ -278,7 +328,7 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) { block libs = gen_noop(); for (uint64_t i = 0; i < lib_state.ct; ++i) { free(lib_state.names[i]);- if (nerrors == 0)+ if (nerrors == 0 && !block_is_const(lib_state.defs[i])) libs = block_join(libs, lib_state.defs[i]); else block_free(lib_state.defs[i]);
main.c
@@ -444,23 +444,11 @@ int main(int argc, char* argv[]) { if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR; if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR; - char *penv = getenv("JQ_LIBRARY_PATH");- if (penv && jv_get_kind(lib_search_paths) == JV_KIND_NULL) {- // Use $JQ_LIBRARY_PATH-#ifdef WIN32-#define PATH_ENV_SEPARATOR ";"-#else-#define PATH_ENV_SEPARATOR ":"-#endif- lib_search_paths = jv_array_concat(lib_search_paths,jv_string_split(jv_string(penv),jv_string(PATH_ENV_SEPARATOR)));-#undef PATH_ENV_SEPARATOR- } else if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) {- // Use compiled-in default JQ_LIBRARY_PATH-#ifdef WIN32- lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/lib"));-#else- lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/../lib/jq"));-#endif+ if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) {+ // Default search path list+ lib_search_paths = JV_ARRAY(jv_string("~/.jq"),+ jv_string("$ORIGIN/../lib/jq"),+ jv_string("$ORIGIN/lib")); } jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths);
opcode_list.h
@@ -6,6 +6,7 @@ OP(POP, NONE, 1, 0) OP(LOADV, VARIABLE, 1, 1) OP(LOADVN, VARIABLE, 1, 1) OP(STOREV, VARIABLE, 1, 0)+OP(STORE_GLOBAL, GLOBAL, 0, 0) OP(INDEX, NONE, 2, 1) OP(INDEX_OPT, NONE, 2, 1) OP(EACH, NONE, 1, 1)
parser.y
@@ -280,14 +280,13 @@ Module: %empty { $$ = gen_noop(); } |-"module" IDENT Exp ';' {- if (!block_is_const($3)) {+"module" Exp ';' {+ if (!block_is_const($2)) { FAIL(@$, "Module metadata must be constant."); $$ = gen_noop(); } else {- $$ = gen_module(jv_string_value($2), $3);+ $$ = gen_module($2); }- jv_free($2); } Imports:@@ -479,33 +478,45 @@ Term { } Import:-"import" IDENT ';' {- $$ = gen_import(jv_string_value($2), gen_noop(), NULL);- jv_free($2);+"import" String "as" '$' IDENT ';' {+ jv v = block_const($2);+ // XXX Make gen_import take only blocks and the int is_data so we+ // don't have to free so much stuff here+ $$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($5), 1);+ block_free($2);+ jv_free($5);+ jv_free(v); } |-"import" IDENT Exp ';' {- if (!block_is_const($3)) {+"import" String "as" IDENT ';' {+ jv v = block_const($2);+ $$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($4), 0);+ block_free($2);+ jv_free($4);+ jv_free(v);+} |+"import" String "as" IDENT Exp ';' {+ if (!block_is_const($5)) { FAIL(@$, "Module metadata must be constant."); $$ = gen_noop(); } else {- $$ = gen_import(jv_string_value($2), $3, NULL);+ jv v = block_const($2);+ $$ = gen_import(jv_string_value(v), $5, jv_string_value($4), 0);+ jv_free(v); }- jv_free($2);-} |-"import" IDENT "as" IDENT ';' {- $$ = gen_import(jv_string_value($2), gen_noop(), jv_string_value($4));- jv_free($2);+ block_free($2); jv_free($4); } |-"import" IDENT "as" IDENT Exp ';' {- if (!block_is_const($5)) {+"import" String "as" '$' IDENT Exp ';' {+ if (!block_is_const($6)) { FAIL(@$, "Module metadata must be constant."); $$ = gen_noop(); } else {- $$ = gen_import(jv_string_value($2), $5, jv_string_value($4));+ jv v = block_const($2);+ $$ = gen_import(jv_string_value(v), $6, jv_string_value($5), 1);+ jv_free(v); }- jv_free($2);- jv_free($4);+ block_free($2);+ jv_free($5); } FuncDef:
@@ -1,2 +1,2 @@-module a {version:1.7};+module {version:1.7}; def a: "a";
@@ -1,12 +1,14 @@-module c {whatever:null};-import a as foo;-import d as d {search:"./"};-import d {search:"./"};-import e as e {search:"./../lib/jq"};-import f as f {search:"./../lib/jq"};+module {whatever:null};+import "a" as foo;+import "d" as d {search:"./"};+import "d" as d2{search:"./"};+import "e" as e {search:"./../lib/jq"};+import "f" as f {search:"./../lib/jq"};+import "data" as $d; def a: 0; def c:- if meh != d::meh then error("import into namespace doesn't work")+ if $d::d[0] != {this:"is a test",that:"is too"} then error("data import is busted")+ elif d2::meh != d::meh then error("import twice doesn't work") elif foo::a != "a" then error("foo::a didn't work as expected") elif d::meh != "meh" then error("d::meh didn't work as expected") elif e::bah != "bah" then error("e::bah didn't work as expected")
tests/run
@@ -159,7 +159,7 @@ if which seq > /dev/null 2>&1; then fi dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |- $VALGRIND ./jq -cn --stream -L "$mods" 'import streaming; tovalues(inputs)' > $d/out1 2>$d/err || true+ $VALGRIND ./jq -cn --stream -L "$mods" 'import "streaming" as streaming; streaming::tovalues(inputs)' > $d/out1 2>$d/err || true if [ -n "$VALGRIND" ]; then grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null else@@ -200,22 +200,22 @@ if [ `HOME="$mods" $VALGRIND $Q ./jq --debug-dump-disasm -n fg | grep '^[a-z]' | exit 1 fi -if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import a as foo; import b as bar; import a; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a, a] | . == ["a","b","c","a","a"]' > /dev/null; then+if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import "a" as foo; import "b" as bar; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a] | . == ["a","b","c","a"]' > /dev/null; then echo "Module system appears to be broken" 1>&2 exit 1 fi -if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import c as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then+if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import "c" as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then echo "Module system appears to be broken" 1>&2 exit 1 fi -if [ "`$VALGRIND $Q ./jq -cner -L "$mods" '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"./","as":"d","name":"d"},{"search":"./","name":"d"},{"search":"./../lib/jq","as":"e","name":"e"},{"search":"./../lib/jq","as":"f","name":"f"}]}' ]; then+if [ "`$VALGRIND $Q ./jq -cner -L "$mods" '\"c\" | modulemeta'`" != '{"whatever":null,"deps":[{"as":"foo","is_data":false,"relpath":"a"},{"search":"./","as":"d","is_data":false,"relpath":"d"},{"search":"./","as":"d2","is_data":false,"relpath":"d"},{"search":"./../lib/jq","as":"e","is_data":false,"relpath":"e"},{"search":"./../lib/jq","as":"f","is_data":false,"relpath":"f"},{"as":"d","is_data":true,"relpath":"data"}]}' ]; then echo "modulemeta builtin appears to be broken" 1>&2 exit 1 fi -if $VALGRIND ./jq -ner -L "$mods" 'import syntaxerror; .' > $d/out 2>&1; then+if $VALGRIND ./jq -ner -L "$mods" 'import "syntaxerror" as e; .' > $d/out 2>&1; then echo "Module system appears to be broken" 1>&2 exit 1 fi