Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ Note that the tests are meant to be used with jq 1.7.1.
- [x] `getpath(path)` (passthrough)
- [x] `group`, `group_by(f)`
- [x] `gsub($regex; f)` (passthrough)
- [ ] `gsub($regex; f; $flags)`
- [x] `gsub($regex; f; $flags)` (passthrough)
- [x] `halt_error`, `halt_error($exit_code)`
- [x] `has($key)` (passthrough)
- [x] `implode` (passthrough)
Expand Down Expand Up @@ -201,12 +201,37 @@ Note that the tests are meant to be used with jq 1.7.1.
- [ ] `@format "string"` Format string
- [ ] `label $out | break $out` Break out
- [ ] `include "f"`, `import "f"` Include
- [ ] CLI options
- [x] `--help` / `-h`
- [x] `--null-input` / `-n`
- [ ] `--raw-input` / `-R`
- [ ] `--compact-output` / `-c`
- [ ] `--raw-output` / `-r`
- [ ] `--raw-output0`
- [ ] `--join-output` / `-j`
- [x] `--slurp` / `-s`
- [x] `--color-output` / `-C`
- [x] `--monochrome-output` / `-M`
- [ ] `-L directory`
- [ ] `--arg name value`
- [ ] `--rawfile name filename`
- [x] `--run-tests`
- [ ] `--run-tests [filename]`
- [x] `--`
- [ ] Combined short options
- [ ] More...
- Non-standard CLI options
- [x] `--jq`
- [x] `--lex`
- [x] `--no-builtins`
- [x] `--parse`
- [x] `--repl`
- [x] Run jqjq with jqjq
- [x] Bugs

### jq's test suite

```
```sh
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to rerun this, still 307, but i did notice that something seems strange with the output when running tests. Have a look at make test guessing it's the switch to --join-output that require some change?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just changed ./jqjq to replace --join-output with --raw-output for --run-tests.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I get 308 on jq.test from jq origin/main.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Huh made sure i had master also now, still 307, weird, some locale thing? anyways, that's for another time

$ ./jqjq --run-tests < ../jq/tests/jq.test | grep passed
307 of 449 tests passed
```
Expand Down
215 changes: 128 additions & 87 deletions jqjq.jq
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
# MIT License
#
# TODO:
# jq bug with error undefined function (possibly https://github.com/stedolan/jq/issues/2485?)
# ".end" lex, require whitespace/end around ident?
# how test associativity 1|2|3?
# add some term builder helper, _term("TermTypeArray"; {query: ...}) etc?
Expand Down Expand Up @@ -1118,6 +1117,81 @@ def parse:
// error("parse error: \(.)")
);

def _tojson($opts):
# color order: null, false, true, number, string, array, object, field
def _color($id):
if $opts.colors != null then "\u001b[\($opts.colors[$id])m"
else empty end;
def _reset_color:
if $opts.colors != null then "\u001b[0m"
else empty end;
def _wrap_color($id):
if $opts.colors != null then _color($id) + . + _reset_color
else . end;
def _f($opts; $indent):
def _r($prefix):
( type as $t
| if $t == "null" then tojson | _wrap_color(0)
elif $t == "string" then tojson | _wrap_color(4)
elif $t == "number" then tojson | _wrap_color(3)
elif $t == "boolean" then
if . then "true" | _wrap_color(2)
else "false" | _wrap_color(1)
end
elif $t == "array" then
if length == 0 then "[]" | _wrap_color(5)
else
[ _color(5), "[", $opts.compound_newline
, ( [ .[]
| $prefix, $indent
, _r($prefix+$indent)
, _color(5), $opts.array_sep
]
| .[0:-1]
)
, $opts.compound_newline
, $prefix, ("]" | _wrap_color(5))
]
end
elif $t == "object" then
if length == 0 then "{}" | _wrap_color(6)
else
[ _color(6), "{", $opts.compound_newline
, ( [ to_entries[]
| $prefix, $indent, _reset_color
, (.key | tojson | _wrap_color(7))
, ($opts.key_sep | _wrap_color(6))
, (.value | _r($prefix+$indent))
, _color(6), $opts.object_sep
]
| .[0:-1]
)
, $opts.compound_newline
, $prefix, ("}" | _wrap_color(6))
]
end
else _internal_error("unknown type \($t)")
end
);
_r("");
( ( { indent: 0,
key_sep: ":",
object_sep: ",",
array_sep: ",",
compound_newline: "",
} + $opts
| if .indent > 0 then
( .key_sep = ": "
| .object_sep = ",\n"
| .array_sep = ",\n"
| .compound_newline = "\n"
)
end
) as $o
| _f($o; $o.indent * " ")
| if type == "array" then flatten | join("") end
);
def _tojson: _tojson({});

def undefined_func_error:
error("undefined function \(.name)");
Expand Down Expand Up @@ -1308,66 +1382,6 @@ def eval_ast($query; $path; $env; undefined_func):
catch
error("fromjson only supports constant literals");

def _tojson($opts):
def _f($opts; $indent):
def _r($prefix):
( type as $t
| if $t == "null" then tojson
elif $t == "string" then tojson
elif $t == "number" then tojson
elif $t == "boolean" then tojson
elif $t == "array" then
if length == 0 then "[]"
else
[ "[", $opts.compound_newline
, ( [ .[]
| $prefix, $indent
, _r($prefix+$indent), $opts.array_sep
]
| .[0:-1]
)
, $opts.compound_newline
, $prefix, "]"
]
end
elif $t == "object" then
if length == 0 then "{}"
else
[ "{", $opts.compound_newline
, ( [ to_entries[]
| $prefix, $indent
, (.key | tojson), $opts.key_sep
, (.value | _r($prefix+$indent)), $opts.object_sep
]
| .[0:-1]
)
, $opts.compound_newline
, $prefix, "}"
]
end
else _internal_error("unknown type \($t)")
end
);
_r("");
( ( { indent: 0,
key_sep: ":",
object_sep: ",",
array_sep: ",",
compound_newline: "",
} + $opts
| if .indent > 0 then
( .key_sep = ": "
| .object_sep = ",\n"
| .array_sep = ",\n"
| .compound_newline = "\n"
)
end
) as $o
| _f($o; $o.indent * " ")
| if type == "array" then flatten | join("") end
);
def _tojson: _tojson({});

( . as $input
| $query.term.func as {$name, $args}
| func_name($name; $args) as $name
Expand Down Expand Up @@ -1447,7 +1461,6 @@ def eval_ast($query; $path; $env; undefined_func):
# TODO: implement in jqjq?
elif $name == "tostring/0" then [[null], tostring]
elif $name == "tojson/0" then [[null], _tojson]
elif $name == "tojson/1" then [[null], _tojson(a0)]
elif $name == "fromjson/0" then [[null], _fromjson]
# TODO: make args general
# note "null | error" is same as empty
Expand Down Expand Up @@ -1521,6 +1534,7 @@ def eval_ast($query; $path; $env; undefined_func):
elif $name == "match/2" then match(a0; a1) | [[null], .]
elif $name == "test/2" then test(a0; a1) | [[null], .]
elif $name == "gsub/2" then gsub(a0; a1) | [[null], .]
elif $name == "gsub/3" then gsub(a0; a1; a2) | [[null], .]
elif $name == "atan2/2" then [[null], atan2(a0; a1)]
elif $name == "copysign/2" then [[null], copysign(a0; a1)]
elif $name == "drem/2" then [[null], drem(a0; a1)]
Expand Down Expand Up @@ -2456,35 +2470,60 @@ def fromjqtest:
def jqjq($args; $env):
def _parse_args:
def _f:
.[0] as $a |
if length == 0 then empty
elif .[0] == "-h" or .[0] == "--help" then {help: true}, (.[1:] | _f)
elif .[0] == "--jq" then {jq: .[1]}, (.[2:] | _f)
elif .[0] == "--lex" then {lex: true}, (.[1:] | _f)
elif .[0] == "--no-builtins" then {no_builtins: true}, (.[1:] | _f)
elif .[0] == "-n" or .[0] == "--null-input" then {null_input: true}, (.[1:] | _f)
elif .[0] == "--parse" then {parse: true}, (.[1:] | _f)
elif .[0] == "--repl" then {repl: true}, (.[1:] | _f)
elif .[0] == "--run-tests" then {run_tests: true}, (.[1:] | _f)
elif .[0] == "-s" or .[0] == "--slurp" then {slurp: true}, (.[1:] | _f)
elif .[0] == "--" then {filter: .[1]}, (.[2:] | _f)
elif .[0] | startswith("-") then error("unknown argument: \(.[0])")
else {filter: .[0]}, (.[1:] | _f)
elif $a == "-h" or $a == "--help" then {help: true}, (.[1:] | _f)
elif $a == "--jq" then {jq: .[1]}, (.[2:] | _f)
elif $a == "--lex" then {lex: true}, (.[1:] | _f)
elif $a == "--no-builtins" then {no_builtins: true}, (.[1:] | _f)
elif $a == "-n" or $a == "--null-input" then {null_input: true}, (.[1:] | _f)
elif $a == "--parse" then {parse: true}, (.[1:] | _f)
elif $a == "--repl" then {repl: true}, (.[1:] | _f)
elif $a == "--run-tests" then {run_tests: true}, (.[1:] | _f)
elif $a == "-s" or $a == "--slurp" then {slurp: true}, (.[1:] | _f)
elif $a == "-C" or $a == "--color-output" then {color_output: true}, (.[1:] | _f)
elif $a == "-M" or $a == "--monochrome-output" then {monochrome_output: true}, (.[1:] | _f)
elif $a == "--" then {filter: .[1]}, (.[2:] | _f)
elif $a | startswith("-") then error("unknown argument: \($a)")
else {filter: $a}, (.[1:] | _f)
end;
( [_f]
| add
);

# get the ANSI color codes for printing values
# corresponds to jv_set_colors and its usage in main
def parse_colors($opts; $env):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fq lacks JQ_COLORS support atm (has it's own color config with more stuff) maybe can steal some of this code :)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're welcome to use it for that :). It should represent the C parsing and formatting exactly.

From my tests, it appears the gojq ignores JQ_COLORS and it formats differently. Maybe it would be worth coordinating a decision with gojq? I figure you probably inherited that from using gojq as a library.

I submitted jqlang/jq#3034, which fixes some usually-invisible inconsistencies with colors in jq. I think that should be merged regardless, but in addition it might worth bringing jq more into line with gojq. In particular, I don't really like how in jq, {, :, ,, and } in objects get the object color and [, ,, and ] in arrays get the array color. I think : and , punctuation should be colored uniformly across objects and arrays. Personally, I don't see much use in coloring braces and brackets, but many people like coloring them by nesting depth in VS Code, so I can see why this simplistic differentiation could be somewhat useful.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're welcome to use it for that :). It should represent the C parsing and formatting exactly.

👍

From my tests, it appears the gojq ignores JQ_COLORS and it formats differently. Maybe it would be worth coordinating a decision with gojq? I figure you probably inherited that from using gojq as a library.

Kind-of, fq uses a modified version of cli/encoder.go from gojq https://github.com/wader/fq/blob/master/internal/colorjson/encoder.go that makes the options non-global and adds some things needed by fq (how to format binary strings etc)

I noticed now that gojq seems to only read GOJQ_COLORS, was thinking if i added support in fq i would probably read FQ_COLORS and fallback to JQ_COLORS? hmm

I submitted jqlang/jq#3034, which fixes some usually-invisible inconsistencies with colors in jq. I think that should be merged regardless, but in addition it might worth bringing jq more into line with gojq. In particular, I don't really like how in jq, {, :, ,, and } in objects get the object color and [, ,, and ] in arrays get the array color. I think : and , punctuation should be colored uniformly across objects and arrays. Personally, I don't see much use in coloring braces and brackets, but many people like coloring them by nesting depth in VS Code, so I can see why this simplistic differentiation could be somewhat useful.

I see, hmm haven't really thought about it much so no string opinion, but making them more consistent make sense i think. Let's see what the other maintainers think

# color order: null, false, true, number, string, array, object, field
( ["0;90", "0;39", "0;39", "0;39", "0;32", "1;39", "1;39", "1;34"] as $default
| if $env | has("JQ_COLORS") then
( ($env.JQ_COLORS | split(":")[:8]) as $custom
| if $custom | all(length <= 12 and test("^[0-9;]*$")) then
$custom + $default[$custom | length:]
else "Failed to set $JQ_COLORS\n" | stderr | $default
end
)
else $default
end
| if $opts.monochrome_output or
(($opts.color_output | not) and ($env.NO_COLOR | . != null and . != "")) then
null
end
);

def _help:
( "jqjq - jq implementation of jq"
, "Usage: jqjq [OPTIONS] [--] [EXPR]"
, " --jq PATH jq implementation to run with"
, " --lex Lex EXPR"
, " --no-builtins Don't include builtins"
, " --null-input,-n Null input"
, " --parse Lex then parse EXPR"
, " --repl REPL"
, " --run-tests Run jq tests from stdin"
, " --slurp,-s Slurp inputs into an array"
, " --jq PATH jq implementation to run with"
, " --lex Lex EXPR"
, " --no-builtins Don't include builtins"
, " --null-input / -n Null input"
, " --parse Lex then parse EXPR"
, " --repl REPL"
, " --run-tests Run jq tests from stdin"
, " --slurp / -s Slurp inputs into an array"
, " --color-output / -C Force colored output"
, " --monochrome-output / -M Disable colored output"
);

def _repl:
Expand All @@ -2494,7 +2533,8 @@ def jqjq($args; $env):
if . == "break" then empty
else error
end;
( builtins_env as $builtins_env
( parse_colors({}; $env) as $colors
| builtins_env as $builtins_env
| _repeat_break(
( "> "
, ( try input
Expand All @@ -2503,7 +2543,7 @@ def jqjq($args; $env):
| null
| try
( eval($expr; {"$ENV": $env}; $builtins_env)
| tojson
| _tojson({$colors})
, "\n"
)
catch
Expand Down Expand Up @@ -2584,7 +2624,7 @@ def jqjq($args; $env):
};
if ($l | type) == "object" then
( .line = false
| if $l.error then .errors +=1
| if $l.error then .errors += 1
elif $l.ok then .oks += 1
elif $l.end then .end = true
else .
Expand Down Expand Up @@ -2612,13 +2652,14 @@ def jqjq($args; $env):
elif $opts.slurp then [inputs]
else inputs
end;
( if $opts.no_builtins then {}
parse_colors($opts; $env) as $colors
| ( if $opts.no_builtins then {}
else builtins_env
end
) as $builtins_env
| _inputs
| eval($opts.filter; {"$ENV": $env}; $builtins_env)
| tojson
| _tojson({$colors})
);

( ( { filter: "."
Expand Down