From fb9def0301fc1e8425f055b42a6045a5fa41a864 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 15:10:57 -0800 Subject: [PATCH 01/15] Remove obsolete TODO --- jqjq.jq | 1 - 1 file changed, 1 deletion(-) diff --git a/jqjq.jq b/jqjq.jq index 2a882f2..7a0e590 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -3,7 +3,6 @@ # MIT License # # TODO: -# jq bug with error undefined function (possibly https://github.com/stedolan/jq/issues/2485?) # ".end" lex, require whitespace/end around ident? # how test associativity 1|2|3? # add some term builder helper, _term("TermTypeArray"; {query: ...}) etc? From d4d18066eb8ee2bcfdffd25c1dba34cc75b365e0 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 15:28:31 -0800 Subject: [PATCH 02/15] Format with JQ_COLORS in _tojson --- jqjq.jq | 61 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index 7a0e590..d05d514 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1308,17 +1308,25 @@ def eval_ast($query; $path; $env; undefined_func): error("fromjson only supports constant literals"); def _tojson($opts): + # color order: null, false, true, number, string, array, object + def _color($id): + if $opts.colors != null then + "\u001b[\($opts.colors[$id])m" + . + "\u001b[0m" + end; def _f($opts; $indent): def _r($prefix): ( type as $t - | if $t == "null" then tojson - elif $t == "string" then tojson - elif $t == "number" then tojson - elif $t == "boolean" then tojson + | if $t == "null" then tojson | _color(0) + elif $t == "string" then tojson | _color(4) + elif $t == "number" then tojson | _color(3) + elif $t == "boolean" then + if . then "true" | _color(2) + else "false" | _color(1) + end elif $t == "array" then - if length == 0 then "[]" + if length == 0 then "[]" | _color(5) else - [ "[", $opts.compound_newline + [ ("[" | _color(5)), $opts.compound_newline , ( [ .[] | $prefix, $indent , _r($prefix+$indent), $opts.array_sep @@ -1326,13 +1334,13 @@ def eval_ast($query; $path; $env; undefined_func): | .[0:-1] ) , $opts.compound_newline - , $prefix, "]" + , $prefix, ("]" | _color(5)) ] end elif $t == "object" then - if length == 0 then "{}" + if length == 0 then "{}" | _color(6) else - [ "{", $opts.compound_newline + [ ("{" | _color(6)), $opts.compound_newline , ( [ to_entries[] | $prefix, $indent , (.key | tojson), $opts.key_sep @@ -1341,7 +1349,7 @@ def eval_ast($query; $path; $env; undefined_func): | .[0:-1] ) , $opts.compound_newline - , $prefix, "}" + , $prefix, ("}" | _color(6)) ] end else _internal_error("unknown type \($t)") @@ -2473,6 +2481,28 @@ def jqjq($args; $env): | add ); + # get the ANSI color codes for printing values + # corresponds to jv_set_colors and its usage in main + # TODO: handle -C/--color-output and -M/--monochrome-output + def parse_colors($env): + # color order: null, false, true, number, string, array, object + ( ["0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"] as $default + | if $env | has("JQ_COLORS") then + ( ($env.JQ_COLORS | split(":")[:8]) as $custom + | if $custom | all(length <= 12 and test("^[0-9;]*$")) then + {colors: ($custom + $default[$custom | length:]), ok: true} + else + {colors: $default, ok: false} + end + ) + else + {colors: $default, ok: true} + end + | if $env | .NO_COLOR != null and .NO_COLOR != "" then + .colors = null + end + ); + def _help: ( "jqjq - jq implementation of jq" , "Usage: jqjq [OPTIONS] [--] [EXPR]" @@ -2493,7 +2523,8 @@ def jqjq($args; $env): if . == "break" then empty else error end; - ( builtins_env as $builtins_env + ( parse_colors($env) as {$colors} + | builtins_env as $builtins_env | _repeat_break( ( "> " , ( try input @@ -2502,6 +2533,7 @@ def jqjq($args; $env): | null | try ( eval($expr; {"$ENV": $env}; $builtins_env) + # TODO: apply colors | tojson , "\n" ) @@ -2583,7 +2615,7 @@ def jqjq($args; $env): }; if ($l | type) == "object" then ( .line = false - | if $l.error then .errors +=1 + | if $l.error then .errors += 1 elif $l.ok then .oks += 1 elif $l.end then .end = true else . @@ -2611,12 +2643,15 @@ def jqjq($args; $env): elif $opts.slurp then [inputs] else inputs end; - ( if $opts.no_builtins then {} + # TODO: jq prints "Failed to set $JQ_COLORS\n" to stderr on failure + parse_colors($env) as {$colors} + | ( if $opts.no_builtins then {} else builtins_env end ) as $builtins_env | _inputs | eval($opts.filter; {"$ENV": $env}; $builtins_env) + # TODO: apply colors | tojson ); From feb9cc0771b497518b2977269b2f71fb10ad1fa6 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 16:20:43 -0800 Subject: [PATCH 03/15] Write to stderr on invalid JQ_COLORS --- jqjq.jq | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index d05d514..028d0f0 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -2490,16 +2490,14 @@ def jqjq($args; $env): | if $env | has("JQ_COLORS") then ( ($env.JQ_COLORS | split(":")[:8]) as $custom | if $custom | all(length <= 12 and test("^[0-9;]*$")) then - {colors: ($custom + $default[$custom | length:]), ok: true} - else - {colors: $default, ok: false} + $custom + $default[$custom | length:] + else "Failed to set $JQ_COLORS\n" | stderr | $default end ) - else - {colors: $default, ok: true} + else $default end | if $env | .NO_COLOR != null and .NO_COLOR != "" then - .colors = null + null end ); @@ -2523,7 +2521,7 @@ def jqjq($args; $env): if . == "break" then empty else error end; - ( parse_colors($env) as {$colors} + ( parse_colors($env) as $colors | builtins_env as $builtins_env | _repeat_break( ( "> " @@ -2643,8 +2641,7 @@ def jqjq($args; $env): elif $opts.slurp then [inputs] else inputs end; - # TODO: jq prints "Failed to set $JQ_COLORS\n" to stderr on failure - parse_colors($env) as {$colors} + parse_colors($env) as $colors | ( if $opts.no_builtins then {} else builtins_env end From 6b24db9dc16e4a513b169009b462667f32b08ec4 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 20:28:23 -0800 Subject: [PATCH 04/15] Color delimiters exactly like jq --- jqjq.jq | 48 ++++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index 028d0f0..9dc2ba3 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1308,48 +1308,56 @@ def eval_ast($query; $path; $env; undefined_func): error("fromjson only supports constant literals"); def _tojson($opts): - # color order: null, false, true, number, string, array, object + # color order: null, false, true, number, string, array, object, field def _color($id): - if $opts.colors != null then - "\u001b[\($opts.colors[$id])m" + . + "\u001b[0m" - end; + if $opts.colors != null then "\u001b[\($opts.colors[$id])m" + else empty end; + def _reset_color: + if $opts.colors != null then "\u001b[0m" + else empty end; + def _wrap_color($id): + if $opts.colors != null then _color($id) + . + _reset_color + else . end; def _f($opts; $indent): def _r($prefix): ( type as $t - | if $t == "null" then tojson | _color(0) - elif $t == "string" then tojson | _color(4) - elif $t == "number" then tojson | _color(3) + | if $t == "null" then tojson | _wrap_color(0) + elif $t == "string" then tojson | _wrap_color(4) + elif $t == "number" then tojson | _wrap_color(3) elif $t == "boolean" then - if . then "true" | _color(2) - else "false" | _color(1) + if . then "true" | _wrap_color(2) + else "false" | _wrap_color(1) end elif $t == "array" then - if length == 0 then "[]" | _color(5) + if length == 0 then "[]" | _wrap_color(5) else - [ ("[" | _color(5)), $opts.compound_newline + [ _color(5), "[", $opts.compound_newline , ( [ .[] | $prefix, $indent - , _r($prefix+$indent), $opts.array_sep + , _r($prefix+$indent) + , _color(5), $opts.array_sep ] | .[0:-1] ) , $opts.compound_newline - , $prefix, ("]" | _color(5)) + , $prefix, ("]" | _wrap_color(5)) ] end elif $t == "object" then - if length == 0 then "{}" | _color(6) + if length == 0 then "{}" | _wrap_color(6) else - [ ("{" | _color(6)), $opts.compound_newline + [ _color(6), "{", $opts.compound_newline , ( [ to_entries[] - | $prefix, $indent - , (.key | tojson), $opts.key_sep - , (.value | _r($prefix+$indent)), $opts.object_sep + | $prefix, $indent, _reset_color + , (.key | tojson | _wrap_color(7)) + , ($opts.key_sep | _wrap_color(6)) + , (.value | _r($prefix+$indent)) + , _color(6), $opts.object_sep ] | .[0:-1] ) , $opts.compound_newline - , $prefix, ("}" | _color(6)) + , $prefix, ("}" | _wrap_color(6)) ] end else _internal_error("unknown type \($t)") @@ -2485,7 +2493,7 @@ def jqjq($args; $env): # corresponds to jv_set_colors and its usage in main # TODO: handle -C/--color-output and -M/--monochrome-output def parse_colors($env): - # color order: null, false, true, number, string, array, object + # color order: null, false, true, number, string, array, object, field ( ["0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"] as $default | if $env | has("JQ_COLORS") then ( ($env.JQ_COLORS | split(":")[:8]) as $custom From 808b0317a015e836e015aeda14e9e77d78c64d62 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 23:11:24 -0800 Subject: [PATCH 05/15] Remove jqjq-specific tojson/1 intrinsic tojson/1 is not defined by jq or gojq --- jqjq.jq | 1 - 1 file changed, 1 deletion(-) diff --git a/jqjq.jq b/jqjq.jq index 9dc2ba3..9ad8175 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1462,7 +1462,6 @@ def eval_ast($query; $path; $env; undefined_func): # TODO: implement in jqjq? elif $name == "tostring/0" then [[null], tostring] elif $name == "tojson/0" then [[null], _tojson] - elif $name == "tojson/1" then [[null], _tojson(a0)] elif $name == "fromjson/0" then [[null], _fromjson] # TODO: make args general # note "null | error" is same as empty From 231277fd0976189d81e03dbc6e51949a039183d1 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 23:14:46 -0800 Subject: [PATCH 06/15] Use colored _tojson for filter result --- jqjq.jq | 157 +++++++++++++++++++++++++++----------------------------- 1 file changed, 77 insertions(+), 80 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index 9ad8175..2a8e049 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1117,6 +1117,81 @@ def parse: // error("parse error: \(.)") ); +def _tojson($opts): + # color order: null, false, true, number, string, array, object, field + def _color($id): + if $opts.colors != null then "\u001b[\($opts.colors[$id])m" + else empty end; + def _reset_color: + if $opts.colors != null then "\u001b[0m" + else empty end; + def _wrap_color($id): + if $opts.colors != null then _color($id) + . + _reset_color + else . end; + def _f($opts; $indent): + def _r($prefix): + ( type as $t + | if $t == "null" then tojson | _wrap_color(0) + elif $t == "string" then tojson | _wrap_color(4) + elif $t == "number" then tojson | _wrap_color(3) + elif $t == "boolean" then + if . then "true" | _wrap_color(2) + else "false" | _wrap_color(1) + end + elif $t == "array" then + if length == 0 then "[]" | _wrap_color(5) + else + [ _color(5), "[", $opts.compound_newline + , ( [ .[] + | $prefix, $indent + , _r($prefix+$indent) + , _color(5), $opts.array_sep + ] + | .[0:-1] + ) + , $opts.compound_newline + , $prefix, ("]" | _wrap_color(5)) + ] + end + elif $t == "object" then + if length == 0 then "{}" | _wrap_color(6) + else + [ _color(6), "{", $opts.compound_newline + , ( [ to_entries[] + | $prefix, $indent, _reset_color + , (.key | tojson | _wrap_color(7)) + , ($opts.key_sep | _wrap_color(6)) + , (.value | _r($prefix+$indent)) + , _color(6), $opts.object_sep + ] + | .[0:-1] + ) + , $opts.compound_newline + , $prefix, ("}" | _wrap_color(6)) + ] + end + else _internal_error("unknown type \($t)") + end + ); + _r(""); + ( ( { indent: 0, + key_sep: ":", + object_sep: ",", + array_sep: ",", + compound_newline: "", + } + $opts + | if .indent > 0 then + ( .key_sep = ": " + | .object_sep = ",\n" + | .array_sep = ",\n" + | .compound_newline = "\n" + ) + end + ) as $o + | _f($o; $o.indent * " ") + | if type == "array" then flatten | join("") end + ); +def _tojson: _tojson({}); def undefined_func_error: error("undefined function \(.name)"); @@ -1307,82 +1382,6 @@ def eval_ast($query; $path; $env; undefined_func): catch error("fromjson only supports constant literals"); - def _tojson($opts): - # color order: null, false, true, number, string, array, object, field - def _color($id): - if $opts.colors != null then "\u001b[\($opts.colors[$id])m" - else empty end; - def _reset_color: - if $opts.colors != null then "\u001b[0m" - else empty end; - def _wrap_color($id): - if $opts.colors != null then _color($id) + . + _reset_color - else . end; - def _f($opts; $indent): - def _r($prefix): - ( type as $t - | if $t == "null" then tojson | _wrap_color(0) - elif $t == "string" then tojson | _wrap_color(4) - elif $t == "number" then tojson | _wrap_color(3) - elif $t == "boolean" then - if . then "true" | _wrap_color(2) - else "false" | _wrap_color(1) - end - elif $t == "array" then - if length == 0 then "[]" | _wrap_color(5) - else - [ _color(5), "[", $opts.compound_newline - , ( [ .[] - | $prefix, $indent - , _r($prefix+$indent) - , _color(5), $opts.array_sep - ] - | .[0:-1] - ) - , $opts.compound_newline - , $prefix, ("]" | _wrap_color(5)) - ] - end - elif $t == "object" then - if length == 0 then "{}" | _wrap_color(6) - else - [ _color(6), "{", $opts.compound_newline - , ( [ to_entries[] - | $prefix, $indent, _reset_color - , (.key | tojson | _wrap_color(7)) - , ($opts.key_sep | _wrap_color(6)) - , (.value | _r($prefix+$indent)) - , _color(6), $opts.object_sep - ] - | .[0:-1] - ) - , $opts.compound_newline - , $prefix, ("}" | _wrap_color(6)) - ] - end - else _internal_error("unknown type \($t)") - end - ); - _r(""); - ( ( { indent: 0, - key_sep: ":", - object_sep: ",", - array_sep: ",", - compound_newline: "", - } + $opts - | if .indent > 0 then - ( .key_sep = ": " - | .object_sep = ",\n" - | .array_sep = ",\n" - | .compound_newline = "\n" - ) - end - ) as $o - | _f($o; $o.indent * " ") - | if type == "array" then flatten | join("") end - ); - def _tojson: _tojson({}); - ( . as $input | $query.term.func as {$name, $args} | func_name($name; $args) as $name @@ -2538,8 +2537,7 @@ def jqjq($args; $env): | null | try ( eval($expr; {"$ENV": $env}; $builtins_env) - # TODO: apply colors - | tojson + | _tojson({$colors}) , "\n" ) catch @@ -2655,8 +2653,7 @@ def jqjq($args; $env): ) as $builtins_env | _inputs | eval($opts.filter; {"$ENV": $env}; $builtins_env) - # TODO: apply colors - | tojson + | _tojson({$colors}) ); ( ( { filter: "." From c1e41483f3abb7ed046ba34dce10091ff14ce53b Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Tue, 6 Feb 2024 23:50:26 -0800 Subject: [PATCH 07/15] Handle --color-output and --monochrome-output --- jqjq.jq | 55 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index 2a8e049..00bf883 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -2469,19 +2469,22 @@ def fromjqtest: def jqjq($args; $env): def _parse_args: def _f: + .[0] as $a | if length == 0 then empty - elif .[0] == "-h" or .[0] == "--help" then {help: true}, (.[1:] | _f) - elif .[0] == "--jq" then {jq: .[1]}, (.[2:] | _f) - elif .[0] == "--lex" then {lex: true}, (.[1:] | _f) - elif .[0] == "--no-builtins" then {no_builtins: true}, (.[1:] | _f) - elif .[0] == "-n" or .[0] == "--null-input" then {null_input: true}, (.[1:] | _f) - elif .[0] == "--parse" then {parse: true}, (.[1:] | _f) - elif .[0] == "--repl" then {repl: true}, (.[1:] | _f) - elif .[0] == "--run-tests" then {run_tests: true}, (.[1:] | _f) - elif .[0] == "-s" or .[0] == "--slurp" then {slurp: true}, (.[1:] | _f) - elif .[0] == "--" then {filter: .[1]}, (.[2:] | _f) - elif .[0] | startswith("-") then error("unknown argument: \(.[0])") - else {filter: .[0]}, (.[1:] | _f) + elif $a == "-h" or $a == "--help" then {help: true}, (.[1:] | _f) + elif $a == "--jq" then {jq: .[1]}, (.[2:] | _f) + elif $a == "--lex" then {lex: true}, (.[1:] | _f) + elif $a == "--no-builtins" then {no_builtins: true}, (.[1:] | _f) + elif $a == "-n" or $a == "--null-input" then {null_input: true}, (.[1:] | _f) + elif $a == "--parse" then {parse: true}, (.[1:] | _f) + elif $a == "--repl" then {repl: true}, (.[1:] | _f) + elif $a == "--run-tests" then {run_tests: true}, (.[1:] | _f) + elif $a == "-s" or $a == "--slurp" then {slurp: true}, (.[1:] | _f) + elif $a == "-C" or $a == "--color-output" then {color_output: true}, (.[1:] | _f) + elif $a == "-M" or $a == "--monochrome-output" then {monochrome_output: true}, (.[1:] | _f) + elif $a == "--" then {filter: .[1]}, (.[2:] | _f) + elif $a | startswith("-") then error("unknown argument: \($a)") + else {filter: $a}, (.[1:] | _f) end; ( [_f] | add @@ -2489,8 +2492,7 @@ def jqjq($args; $env): # get the ANSI color codes for printing values # corresponds to jv_set_colors and its usage in main - # TODO: handle -C/--color-output and -M/--monochrome-output - def parse_colors($env): + def parse_colors($opts; $env): # color order: null, false, true, number, string, array, object, field ( ["0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"] as $default | if $env | has("JQ_COLORS") then @@ -2502,7 +2504,8 @@ def jqjq($args; $env): ) else $default end - | if $env | .NO_COLOR != null and .NO_COLOR != "" then + | if $opts.monochrome_output or + (($opts.color_output | not) and ($env.NO_COLOR | . != null and . != "")) then null end ); @@ -2510,14 +2513,16 @@ def jqjq($args; $env): def _help: ( "jqjq - jq implementation of jq" , "Usage: jqjq [OPTIONS] [--] [EXPR]" - , " --jq PATH jq implementation to run with" - , " --lex Lex EXPR" - , " --no-builtins Don't include builtins" - , " --null-input,-n Null input" - , " --parse Lex then parse EXPR" - , " --repl REPL" - , " --run-tests Run jq tests from stdin" - , " --slurp,-s Slurp inputs into an array" + , " --jq PATH jq implementation to run with" + , " --lex Lex EXPR" + , " --no-builtins Don't include builtins" + , " --null-input / -n Null input" + , " --parse Lex then parse EXPR" + , " --repl REPL" + , " --run-tests Run jq tests from stdin" + , " --slurp / -s Slurp inputs into an array" + , " --color-output / -C Force colored output" + , " --monochrome-output / -M Disable colored output" ); def _repl: @@ -2527,7 +2532,7 @@ def jqjq($args; $env): if . == "break" then empty else error end; - ( parse_colors($env) as $colors + ( parse_colors({}; $env) as $colors | builtins_env as $builtins_env | _repeat_break( ( "> " @@ -2646,7 +2651,7 @@ def jqjq($args; $env): elif $opts.slurp then [inputs] else inputs end; - parse_colors($env) as $colors + parse_colors($opts; $env) as $colors | ( if $opts.no_builtins then {} else builtins_env end From d7cea3ca5892c41c0d09bfbb67e9ca728fbac934 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 00:48:02 -0800 Subject: [PATCH 08/15] Include CLI options in progress; add gsub/3 --- README.md | 29 +++++++++++++++++++++++++++-- jqjq.jq | 1 + 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b74a42..5a22b57 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Note that the tests are meant to be used with jq 1.7.1. - [x] `getpath(path)` (passthrough) - [x] `group`, `group_by(f)` - [x] `gsub($regex; f)` (passthrough) - - [ ] `gsub($regex; f; $flags)` + - [x] `gsub($regex; f; $flags)` (passthrough) - [x] `halt_error`, `halt_error($exit_code)` - [x] `has($key)` (passthrough) - [x] `implode` (passthrough) @@ -201,12 +201,37 @@ Note that the tests are meant to be used with jq 1.7.1. - [ ] `@format "string"` Format string - [ ] `label $out | break $out` Break out - [ ] `include "f"`, `import "f"` Include +- [ ] CLI options + - [x] `--help` / `-h` + - [x] `--null-input` / `-n` + - [ ] `--raw-input` / `-R` + - [ ] `--compact-output` / `-c` + - [ ] `--raw-output` / `-r` + - [ ] `--raw-output0` + - [ ] `--join-output` / `-j` + - [x] `--slurp` / `-s` + - [x] `--color-output` / `-C` + - [x] `--monochrome-output` / `-M` + - [ ] `-L directory` + - [ ] `--arg name value` + - [ ] `--rawfile name filename` + - [x] `--run-tests` + - [ ] `--run-tests [filename]` + - [x] `--` + - [ ] Combined short options + - [ ] More... + - Non-standard CLI options + - [x] `--jq` + - [x] `--lex` + - [x] `--no-builtins` + - [x] `--parse` + - [x] `--repl` - [x] Run jqjq with jqjq - [x] Bugs ### jq's test suite -``` +```sh $ ./jqjq --run-tests < ../jq/tests/jq.test | grep passed 307 of 449 tests passed ``` diff --git a/jqjq.jq b/jqjq.jq index 00bf883..d3e0384 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1534,6 +1534,7 @@ def eval_ast($query; $path; $env; undefined_func): elif $name == "match/2" then match(a0; a1) | [[null], .] elif $name == "test/2" then test(a0; a1) | [[null], .] elif $name == "gsub/2" then gsub(a0; a1) | [[null], .] + elif $name == "gsub/3" then gsub(a0; a1; a2) | [[null], .] elif $name == "atan2/2" then [[null], atan2(a0; a1)] elif $name == "copysign/2" then [[null], copysign(a0; a1)] elif $name == "drem/2" then [[null], drem(a0; a1)] From f4487450bb249411ca85c84b812d8cdc12ce3539 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 03:14:37 -0800 Subject: [PATCH 09/15] Add raw output options --- README.md | 6 +++--- jqjq | 4 ++-- jqjq.jq | 29 ++++++++++++++++++++++++----- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5a22b57..306627a 100644 --- a/README.md +++ b/README.md @@ -206,9 +206,9 @@ Note that the tests are meant to be used with jq 1.7.1. - [x] `--null-input` / `-n` - [ ] `--raw-input` / `-R` - [ ] `--compact-output` / `-c` - - [ ] `--raw-output` / `-r` - - [ ] `--raw-output0` - - [ ] `--join-output` / `-j` + - [x] `--raw-output` / `-r` + - [x] `--raw-output0` + - [x] `--join-output` / `-j` - [x] `--slurp` / `-s` - [x] `--color-output` / `-C` - [x] `--monochrome-output` / `-M` diff --git a/jqjq b/jqjq index 15e4862..6fc37ab 100755 --- a/jqjq +++ b/jqjq @@ -6,13 +6,13 @@ JQ="jq" JQJQ_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" ARGS=("$@") -JQ_ARGS=("--raw-output" "--null-input") +JQ_ARGS=("--join-output" "--null-input") # some arguments require to run jq in different input/output mode while [ "$1" != "" ]; do case "$1" in --jq) shift; JQ="$1";; --run-tests) JQ_ARGS+=("--raw-input" "--slurp");; - --repl) JQ_ARGS+=("--raw-input" "--join-output");; + --repl) JQ_ARGS+=("--raw-input");; --) break;; esac shift diff --git a/jqjq.jq b/jqjq.jq index d3e0384..f7e7746 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -2476,13 +2476,19 @@ def jqjq($args; $env): elif $a == "--jq" then {jq: .[1]}, (.[2:] | _f) elif $a == "--lex" then {lex: true}, (.[1:] | _f) elif $a == "--no-builtins" then {no_builtins: true}, (.[1:] | _f) - elif $a == "-n" or $a == "--null-input" then {null_input: true}, (.[1:] | _f) elif $a == "--parse" then {parse: true}, (.[1:] | _f) elif $a == "--repl" then {repl: true}, (.[1:] | _f) - elif $a == "--run-tests" then {run_tests: true}, (.[1:] | _f) + elif $a == "-n" or $a == "--null-input" then {null_input: true}, (.[1:] | _f) + elif $a == "-r" or $a == "--raw-output" then {raw_output: true}, (.[1:] | _f) + elif $a == "--raw-output0" then {raw_output: true, + raw_no_lf: true, + raw_output0: true}, (.[1:] | _f) + elif $a == "-j" or $a == "--join-output" then {raw_output: true, + raw_no_lf: true}, (.[1:] | _f) elif $a == "-s" or $a == "--slurp" then {slurp: true}, (.[1:] | _f) elif $a == "-C" or $a == "--color-output" then {color_output: true}, (.[1:] | _f) elif $a == "-M" or $a == "--monochrome-output" then {monochrome_output: true}, (.[1:] | _f) + elif $a == "--run-tests" then {run_tests: true}, (.[1:] | _f) elif $a == "--" then {filter: .[1]}, (.[2:] | _f) elif $a | startswith("-") then error("unknown argument: \($a)") else {filter: $a}, (.[1:] | _f) @@ -2514,16 +2520,22 @@ def jqjq($args; $env): def _help: ( "jqjq - jq implementation of jq" , "Usage: jqjq [OPTIONS] [--] [EXPR]" + , "" + , "Options:" , " --jq PATH jq implementation to run with" , " --lex Lex EXPR" , " --no-builtins Don't include builtins" - , " --null-input / -n Null input" , " --parse Lex then parse EXPR" , " --repl REPL" - , " --run-tests Run jq tests from stdin" + , "" + , " --null-input / -n Null input" + , " --raw-output / -r Output strings raw with newline" + , " --raw-output0 Output strings raw with NUL" + , " --join-output Output strings raw" , " --slurp / -s Slurp inputs into an array" , " --color-output / -C Force colored output" , " --monochrome-output / -M Disable colored output" + , " --run-tests Run jq tests from stdin" ); def _repl: @@ -2659,7 +2671,14 @@ def jqjq($args; $env): ) as $builtins_env | _inputs | eval($opts.filter; {"$ENV": $env}; $builtins_env) - | _tojson({$colors}) + | if $opts.raw_output and type == "string" then + if $opts.raw_output0 and contains("\u0000") then + error("Cannot dump a string containing NUL with --raw-output0 option") + end + else _tojson({$colors}) + end + | if $opts.raw_no_lf | not then ., "\n" end + | if $opts.raw_output0 then ., "\u0000" end ); ( ( { filter: "." From 843118e6c624253fcfd067038e7e0ce1d6490d75 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 14:20:23 -0800 Subject: [PATCH 10/15] Describe jq logic --- jqjq.jq | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index f7e7746..fc1de9c 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -2498,15 +2498,19 @@ def jqjq($args; $env): ); # get the ANSI color codes for printing values - # corresponds to jv_set_colors and its usage in main + # corresponds to jv_set_colors in jq and its usage in main def parse_colors($opts; $env): # color order: null, false, true, number, string, array, object, field ( ["0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"] as $default | if $env | has("JQ_COLORS") then + # only up to the first 8 color sequences are used ( ($env.JQ_COLORS | split(":")[:8]) as $custom + # jq limits color sequences to 12 bytes, to fit into 16-byte buffers + # with ESC, [, m, and NUL. | if $custom | all(length <= 12 and test("^[0-9;]*$")) then $custom + $default[$custom | length:] - else "Failed to set $JQ_COLORS\n" | stderr | $default + else + "Failed to set $JQ_COLORS\n" | stderr | $default end ) else $default From 0477e3fbe4bfa42461145344bad84a3d72aa405a Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 14:29:37 -0800 Subject: [PATCH 11/15] Use named constants for color IDs --- jqjq.jq | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index fc1de9c..919aa97 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1118,7 +1118,15 @@ def parse: ); def _tojson($opts): - # color order: null, false, true, number, string, array, object, field + # see jq jv_print.c:jv_dump_term for the reference color printing logic + def _c_null: 0; + def _c_false: 1; + def _c_true: 2; + def _c_number: 3; + def _c_string: 4; + def _c_array: 5; + def _c_object: 6; + def _c_field: 7; def _color($id): if $opts.colors != null then "\u001b[\($opts.colors[$id])m" else empty end; @@ -1131,43 +1139,43 @@ def _tojson($opts): def _f($opts; $indent): def _r($prefix): ( type as $t - | if $t == "null" then tojson | _wrap_color(0) - elif $t == "string" then tojson | _wrap_color(4) - elif $t == "number" then tojson | _wrap_color(3) + | if $t == "null" then tojson | _wrap_color(_c_null) + elif $t == "string" then tojson | _wrap_color(_c_string) + elif $t == "number" then tojson | _wrap_color(_c_number) elif $t == "boolean" then - if . then "true" | _wrap_color(2) - else "false" | _wrap_color(1) + if . then "true" | _wrap_color(_c_true) + else "false" | _wrap_color(_c_false) end elif $t == "array" then - if length == 0 then "[]" | _wrap_color(5) + if length == 0 then "[]" | _wrap_color(_c_array) else - [ _color(5), "[", $opts.compound_newline + [ _color(_c_array), "[", $opts.compound_newline , ( [ .[] | $prefix, $indent , _r($prefix+$indent) - , _color(5), $opts.array_sep + , _color(_c_array), $opts.array_sep ] | .[0:-1] ) , $opts.compound_newline - , $prefix, ("]" | _wrap_color(5)) + , $prefix, ("]" | _wrap_color(_c_array)) ] end elif $t == "object" then - if length == 0 then "{}" | _wrap_color(6) + if length == 0 then "{}" | _wrap_color(_c_object) else - [ _color(6), "{", $opts.compound_newline + [ _color(_c_object), "{", $opts.compound_newline , ( [ to_entries[] | $prefix, $indent, _reset_color - , (.key | tojson | _wrap_color(7)) - , ($opts.key_sep | _wrap_color(6)) + , (.key | tojson | _wrap_color(_c_field)) + , ($opts.key_sep | _wrap_color(_c_object)) , (.value | _r($prefix+$indent)) - , _color(6), $opts.object_sep + , _color(_c_object), $opts.object_sep ] | .[0:-1] ) , $opts.compound_newline - , $prefix, ("}" | _wrap_color(6)) + , $prefix, ("}" | _wrap_color(_c_object)) ] end else _internal_error("unknown type \($t)") From 39bd5edb2c4bab52ef6002676b4710c3ddd59771 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 14:59:20 -0800 Subject: [PATCH 12/15] Add --compact-output --- README.md | 2 +- jqjq.jq | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 306627a..baf4c76 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ Note that the tests are meant to be used with jq 1.7.1. - [x] `--help` / `-h` - [x] `--null-input` / `-n` - [ ] `--raw-input` / `-R` - - [ ] `--compact-output` / `-c` + - [x] `--compact-output` / `-c` - [x] `--raw-output` / `-r` - [x] `--raw-output0` - [x] `--join-output` / `-j` diff --git a/jqjq.jq b/jqjq.jq index 919aa97..dedecd0 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -2487,6 +2487,7 @@ def jqjq($args; $env): elif $a == "--parse" then {parse: true}, (.[1:] | _f) elif $a == "--repl" then {repl: true}, (.[1:] | _f) elif $a == "-n" or $a == "--null-input" then {null_input: true}, (.[1:] | _f) + elif $a == "-c" or $a == "--compact-output" then {compact_output: true}, (.[1:] | _f) elif $a == "-r" or $a == "--raw-output" then {raw_output: true}, (.[1:] | _f) elif $a == "--raw-output0" then {raw_output: true, raw_no_lf: true, @@ -2507,7 +2508,7 @@ def jqjq($args; $env): # get the ANSI color codes for printing values # corresponds to jv_set_colors in jq and its usage in main - def parse_colors($opts; $env): + def _parse_colors($opts; $env): # color order: null, false, true, number, string, array, object, field ( ["0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"] as $default | if $env | has("JQ_COLORS") then @@ -2529,6 +2530,13 @@ def jqjq($args; $env): end ); + def _parse_opts($opts; $env): + ( $opts + + { colors: _parse_colors($opts; $env) + , indent: (if $opts.compact_output then 0 else 2 end) + } + ); + def _help: ( "jqjq - jq implementation of jq" , "Usage: jqjq [OPTIONS] [--] [EXPR]" @@ -2541,6 +2549,7 @@ def jqjq($args; $env): , " --repl REPL" , "" , " --null-input / -n Null input" + , " --compact-output / -c Output each object on one line" , " --raw-output / -r Output strings raw with newline" , " --raw-output0 Output strings raw with NUL" , " --join-output Output strings raw" @@ -2557,7 +2566,7 @@ def jqjq($args; $env): if . == "break" then empty else error end; - ( parse_colors({}; $env) as $colors + ( _parse_opts({}; $env) as $opts | builtins_env as $builtins_env | _repeat_break( ( "> " @@ -2567,7 +2576,7 @@ def jqjq($args; $env): | null | try ( eval($expr; {"$ENV": $env}; $builtins_env) - | _tojson({$colors}) + | _tojson($opts) , "\n" ) catch @@ -2676,7 +2685,7 @@ def jqjq($args; $env): elif $opts.slurp then [inputs] else inputs end; - parse_colors($opts; $env) as $colors + _parse_opts($opts; $env) as $opts | ( if $opts.no_builtins then {} else builtins_env end @@ -2687,7 +2696,7 @@ def jqjq($args; $env): if $opts.raw_output0 and contains("\u0000") then error("Cannot dump a string containing NUL with --raw-output0 option") end - else _tojson({$colors}) + else _tojson($opts) end | if $opts.raw_no_lf | not then ., "\n" end | if $opts.raw_output0 then ., "\u0000" end From 5a743555a0d0990c0d2f3a3893b74babf0ebb6e1 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 15:31:32 -0800 Subject: [PATCH 13/15] Consistently reset colors to match jq Updates to match the changes in https://github.com/jqlang/jq/pull/3034. --- jqjq.jq | 63 +++++++++++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/jqjq.jq b/jqjq.jq index dedecd0..62561e8 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -1128,71 +1128,62 @@ def _tojson($opts): def _c_object: 6; def _c_field: 7; def _color($id): - if $opts.colors != null then "\u001b[\($opts.colors[$id])m" - else empty end; - def _reset_color: - if $opts.colors != null then "\u001b[0m" - else empty end; - def _wrap_color($id): - if $opts.colors != null then _color($id) + . + _reset_color + if $opts.colors != null then + "\u001b[\($opts.colors[$id])m" + . + "\u001b[0m" else . end; def _f($opts; $indent): def _r($prefix): ( type as $t - | if $t == "null" then tojson | _wrap_color(_c_null) - elif $t == "string" then tojson | _wrap_color(_c_string) - elif $t == "number" then tojson | _wrap_color(_c_number) + | if $t == "null" then tojson | _color(_c_null) + elif $t == "string" then tojson | _color(_c_string) + elif $t == "number" then tojson | _color(_c_number) elif $t == "boolean" then - if . then "true" | _wrap_color(_c_true) - else "false" | _wrap_color(_c_false) + if . then "true" | _color(_c_true) + else "false" | _color(_c_false) end elif $t == "array" then - if length == 0 then "[]" | _wrap_color(_c_array) + if length == 0 then "[]" | _color(_c_array) else - [ _color(_c_array), "[", $opts.compound_newline + [ ("[" | _color(_c_array)), $opts.newline , ( [ .[] | $prefix, $indent , _r($prefix+$indent) - , _color(_c_array), $opts.array_sep + , ("," | _color(_c_array)), $opts.newline ] - | .[0:-1] + | .[0:-2] ) - , $opts.compound_newline - , $prefix, ("]" | _wrap_color(_c_array)) + , $opts.newline + , $prefix, ("]" | _color(_c_array)) ] end elif $t == "object" then - if length == 0 then "{}" | _wrap_color(_c_object) + if length == 0 then "{}" | _color(_c_object) else - [ _color(_c_object), "{", $opts.compound_newline + [ ("{" | _color(_c_object)), $opts.newline , ( [ to_entries[] - | $prefix, $indent, _reset_color - , (.key | tojson | _wrap_color(_c_field)) - , ($opts.key_sep | _wrap_color(_c_object)) + | $prefix, $indent + , (.key | tojson | _color(_c_field)) + , (":" | _color(_c_object)), $opts.space , (.value | _r($prefix+$indent)) - , _color(_c_object), $opts.object_sep + , ("," | _color(_c_object)), $opts.newline ] - | .[0:-1] + | .[0:-2] ) - , $opts.compound_newline - , $prefix, ("}" | _wrap_color(_c_object)) + , $opts.newline + , $prefix, ("}" | _color(_c_object)) ] end else _internal_error("unknown type \($t)") end ); _r(""); - ( ( { indent: 0, - key_sep: ":", - object_sep: ",", - array_sep: ",", - compound_newline: "", + ( ( { indent: 0 + , newline: "" + , space: "" } + $opts | if .indent > 0 then - ( .key_sep = ": " - | .object_sep = ",\n" - | .array_sep = ",\n" - | .compound_newline = "\n" + ( .newline = "\n" + | .space = " " ) end ) as $o From 3d715f55c536ba376420814c16d23623a974f308 Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 15:38:31 -0800 Subject: [PATCH 14/15] Match option order of jq manual and `man jq` --- README.md | 2 +- jqjq.jq | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index baf4c76..38f91ae 100644 --- a/README.md +++ b/README.md @@ -205,11 +205,11 @@ Note that the tests are meant to be used with jq 1.7.1. - [x] `--help` / `-h` - [x] `--null-input` / `-n` - [ ] `--raw-input` / `-R` + - [x] `--slurp` / `-s` - [x] `--compact-output` / `-c` - [x] `--raw-output` / `-r` - [x] `--raw-output0` - [x] `--join-output` / `-j` - - [x] `--slurp` / `-s` - [x] `--color-output` / `-C` - [x] `--monochrome-output` / `-M` - [ ] `-L directory` diff --git a/jqjq.jq b/jqjq.jq index 62561e8..1008c98 100644 --- a/jqjq.jq +++ b/jqjq.jq @@ -2478,6 +2478,7 @@ def jqjq($args; $env): elif $a == "--parse" then {parse: true}, (.[1:] | _f) elif $a == "--repl" then {repl: true}, (.[1:] | _f) elif $a == "-n" or $a == "--null-input" then {null_input: true}, (.[1:] | _f) + elif $a == "-s" or $a == "--slurp" then {slurp: true}, (.[1:] | _f) elif $a == "-c" or $a == "--compact-output" then {compact_output: true}, (.[1:] | _f) elif $a == "-r" or $a == "--raw-output" then {raw_output: true}, (.[1:] | _f) elif $a == "--raw-output0" then {raw_output: true, @@ -2485,7 +2486,6 @@ def jqjq($args; $env): raw_output0: true}, (.[1:] | _f) elif $a == "-j" or $a == "--join-output" then {raw_output: true, raw_no_lf: true}, (.[1:] | _f) - elif $a == "-s" or $a == "--slurp" then {slurp: true}, (.[1:] | _f) elif $a == "-C" or $a == "--color-output" then {color_output: true}, (.[1:] | _f) elif $a == "-M" or $a == "--monochrome-output" then {monochrome_output: true}, (.[1:] | _f) elif $a == "--run-tests" then {run_tests: true}, (.[1:] | _f) @@ -2540,11 +2540,11 @@ def jqjq($args; $env): , " --repl REPL" , "" , " --null-input / -n Null input" + , " --slurp / -s Slurp inputs into an array" , " --compact-output / -c Output each object on one line" , " --raw-output / -r Output strings raw with newline" , " --raw-output0 Output strings raw with NUL" , " --join-output Output strings raw" - , " --slurp / -s Slurp inputs into an array" , " --color-output / -C Force colored output" , " --monochrome-output / -M Disable colored output" , " --run-tests Run jq tests from stdin" From 1822bdbb802baa5ec289fdf4d707f11f554b3edc Mon Sep 17 00:00:00 2001 From: Thalia Archibald Date: Wed, 7 Feb 2024 16:04:56 -0800 Subject: [PATCH 15/15] Use --raw-output for tests; use bash from PATH --- jqjq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jqjq b/jqjq index 6fc37ab..a94115f 100755 --- a/jqjq +++ b/jqjq @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # wrapper for jqjq cli entrypoint # shellcheck disable=SC2016 @@ -11,7 +11,7 @@ JQ_ARGS=("--join-output" "--null-input") while [ "$1" != "" ]; do case "$1" in --jq) shift; JQ="$1";; - --run-tests) JQ_ARGS+=("--raw-input" "--slurp");; + --run-tests) JQ_ARGS+=("--raw-input" "--slurp"); JQ_ARGS[0]="--raw-output";; --repl) JQ_ARGS+=("--raw-input");; --) break;; esac