diff --git a/examples/formatjson/formatjson.d b/examples/formatjson/formatjson.d index 52deb95..07762c2 100644 --- a/examples/formatjson/formatjson.d +++ b/examples/formatjson/formatjson.d @@ -7,6 +7,9 @@ import iopipe.textpipe; import iopipe.valve; import iopipe.json.parser; import iopipe.buffer; +import iopipe.json.serialize; +import iopipe.json.dom; +import iopipe.json.formatter; void main(string[] args) { @@ -20,7 +23,7 @@ void main(string[] args) auto outputter = bufd!(char).push!(c => c .encodeText!(UTFType.UTF8) .outputPipe(File(stdout).refCounted)); - + /* int spacing; enum indent = 4; void putIndent() @@ -92,4 +95,60 @@ void main(string[] args) item = parser.next; } putStr("\n"); + */ + + JSONValue!string jsonData; + deserialize(parser, jsonData); + auto formatter = JSONFormatter!(typeof(outputter))(outputter); + + serializeWithFormatter(formatter, jsonData); + formatter.flushWritten(); +} + +void serializeWithFormatter(Formatter, T)(ref Formatter formatter, T val) +{ + static if (is(T == JSONValue!string)) + { + with(JSONType) final switch(val.type) + { + case Obj: + formatter.beginObject(); + foreach(key, value; val.object) + { + formatter.addMember(key); + serializeWithFormatter(formatter, value); + } + formatter.endObject(); + break; + case Array: + formatter.beginArray(); + foreach(item; val.array) + { + formatter.beginArrayValue(); + serializeWithFormatter(formatter, item); + } + formatter.endArray(); + break; + case String: + formatter.beginString(); + formatter.addStringData(val.str); + formatter.endString(); + break; + case Integer: + formatter.addNumericData(val.integer); + break; + case Floating: + formatter.addNumericData(val.floating); + break; + case Bool: + if(val.boolean) + formatter.addKeywordValue(KeywordValue.True); + else + formatter.addKeywordValue(KeywordValue.False); + break; + case Null: + formatter.addKeywordValue(KeywordValue.Null); + break; + } + } } diff --git a/source/iopipe/json/formatter.d b/source/iopipe/json/formatter.d new file mode 100644 index 0000000..1ee43f1 --- /dev/null +++ b/source/iopipe/json/formatter.d @@ -0,0 +1,832 @@ +module iopipe.json.formatter; + +public import iopipe.json.common; + +import iopipe.traits : isIopipe, WindowType; +import iopipe.bufpipe; +import iopipe.json.serialize : jsonEscapeSubstitutions; + +import std.bitmanip : BitArray; +import std.exception : enforce; +import std.traits : isNumeric; +import std.conv : to; +import std.regex; +import std.algorithm.iteration : substitute; +import std.algorithm.searching : startsWith, endsWith; +import std.format; + +struct MemberOptions { + char quoteChar = '"'; // Use " or ' for quotes + bool escapeKey = true; // Whether to escape special chars in key + + enum ColonSpacing { + none, // "key":"value" + after, // "key": "value" + both, // "key" : "value" + } + + ColonSpacing colonSpacing = ColonSpacing.after; // Default spacing +} + +enum KeywordValue : string { + Null = "null", + True = "true", + False = "false", + // JSON5 extensions + Infinity = "Infinity", + NegativeInfinity = "-Infinity", + NaN = "NaN" +} + +struct JSONFormatter(Chain) +if (isIopipe!Chain) +{ +private: + + Chain* outputChain; + int spacing = 0; + int indent = 4; + MemberOptions memberOptions; + + size_t nWritten = 0; + char currentQuoteChar = char.init; + + enum State : ubyte + { + Begin, // next item should be either an Object or Array or String or Number or Keyword + First, // Expect the first member of a new object or array. + /* + * In an object: + * - First + * - Value(key) + * - Value(value) + * + * - Member + * - Value(key) + * - Value(value) + * ... + * + * In an array: + * - First(value) + * - Member(value) + * ... + */ + Member, // Expect next member (key for object, value for array) + Value, // Expect value + End // there shouldn't be any more items + } + + // How to treat string data passed to addStringData + enum StringMode : ubyte + { + passThru, // do not modify or validate the contents + addEscapes, // escape any characters that would be invalid in JSON (default) + validate // validate that existing escapes / characters are JSON-valid + } + + // 0 = array, 1 = object (same convention as parser) + BitArray stack; + size_t stackLen; + State state = State.Begin; + + bool inObj() const @property nothrow + { + return stackLen == 0 ? false : stack[stackLen - 1]; + } + + // Can we add a value (string, number, keyword, begin object/array)? + bool canAddValue() const nothrow + { + if (currentQuoteChar != char.init) + return false; // can't add value inside a string + + final switch (state) with (State) + { + case Begin: + return true; // root value + case First: + // only allowed for arrays; for objects we must go through addMember + return !inObj(); + case Member: + // only allowed for arrays; for objects we must go through addMember + return !inObj(); + case Value: + return true; + case End: + return false; + } + } + + // Can we add an object member (key)? + // If so, add comma/indent as needed. + bool canAddObjMember() + { + if (!inObj()) + return false; + final switch (state) with (State) + { + case First: + return true; + case Member: + putStr(","); + putIndent(); + return true; + case Begin: + case Value: + case End: + return false; + } + } + + // Can we add an array member? + // If so, add comma/indent as needed. + bool canAddArrayMember() + { + if (inObj()) + return false; + final switch (state) with (State) + { + case First: + return true; + case Member: + putStr(","); + putIndent(); + return true; + case Begin: + case Value: + case End: + return false; + } + } + + bool canCloseObject() const nothrow + { + return inObj() && (state == State.Member || state == State.First); + } + + bool canCloseArray() const nothrow + { + return stackLen > 0 && !inObj() && (state == State.Member || state == State.First); + } + + void pushContainer(bool isObj) + { + if (stackLen == stack.length) + stack ~= isObj; + else + stack[stackLen] = isObj; + ++stackLen; + } + + void popContainer() + { + state = (--stackLen == 0) ? State.End : State.Member; + if (state == state.End) + putIndent(); + } + + void putIndent() + { + outputChain.ensureElems(nWritten + indent * spacing + 1); + outputChain.window[nWritten] = '\n'; + outputChain.window[nWritten + 1 .. nWritten + 1 + indent * spacing] = ' '; + nWritten += indent * spacing + 1; + } + + void putStr(const(char)[] s) + { + outputChain.ensureElems(nWritten + s.length); + outputChain.window[nWritten .. nWritten + s.length] = s; + nWritten += s.length; + } + + bool isValidJSONNumber(string str) { + static auto numberRegex = regex(r"^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$"); + return !str.matchFirst(numberRegex).empty; + } + +public: + this(ref Chain chain) { + outputChain = &chain; + } + + this(ref Chain chain, int spacing, int indent) { + outputChain = &chain; + this.spacing = spacing; + this.indent = indent; + } + + void beginObject() { + if (!canAddValue()) + throw new JSONIopipeException("beginObject not allowed in current state"); + + pushContainer(true); // object + state = State.First; // just started + + ++spacing; + putStr("{"); + putIndent(); + } + + void endObject() { + if (!canCloseObject()) + throw new JSONIopipeException("endObject not allowed in current state"); + + --spacing; + putIndent(); + putStr("}"); + + popContainer(); // State.End or State.Member + } + + void beginArray() { + if (!canAddValue()) + throw new JSONIopipeException("beginArray not allowed in current state"); + + pushContainer(false); // array + state = State.First; + + ++spacing; + putStr("["); + putIndent(); + } + + void endArray() { + if (!canCloseArray()) + throw new JSONIopipeException("endArray not allowed in current state"); + + --spacing; + putIndent(); + putStr("]"); + + popContainer(); + } + + // call before each element + void beginArrayValue() + { + if (!canAddArrayMember()) + throw new JSONIopipeException("beginArrayValue not allowed in current state"); + } + + void addMember(T)(T key) { + addMember(key, memberOptions); + } + + // First/Member -> Value(before string key) -> Member(after string key) -> Value(before value) -> Member(after value) ... + void addMember(T)(T key, MemberOptions options) { + + if (!canAddObjMember()) + throw new JSONIopipeException("addMember not allowed in current state"); + + // prepare for a key string + state = State.Value; + + beginString(options.quoteChar); + static if (__traits(compiles, putStr(key))) + { + if (options.escapeKey) + addStringData(key); + else + addStringData!(StringMode.passThru)(key); + } + else + { + putStr(to!string(key)); + } + endString(); + + final switch (options.colonSpacing) + { + case MemberOptions.ColonSpacing.none: + putStr(":"); + break; + case MemberOptions.ColonSpacing.after: + putStr(": "); + break; + case MemberOptions.ColonSpacing.both: + putStr(" : "); + break; + } + + // prepare for a key's value + state = State.Value; + } + + void beginString(char quoteChar = '"') { + if (!canAddValue()) + throw new JSONIopipeException("beginString not allowed in current state"); + + if (currentQuoteChar != char.init) + throw new JSONIopipeException("nested beginString not allowed"); + + if (quoteChar != '"' && quoteChar != '\'') + throw new JSONIopipeException("Invalid quote character"); + + currentQuoteChar = quoteChar; + putStr(("eChar)[0 .. 1]); + } + + void addStringData(StringMode mode = StringMode.addEscapes, T)(T value) { + if (currentQuoteChar == char.init) + throw new JSONIopipeException("cannot add string data outside of beginString/endString"); + + static if (mode == StringMode.validate) + { + import std.ascii : isHexDigit; + + auto s = value.to!string; // ensure we can index / look ahead + + size_t i = 0; + while (i < s.length) + { + immutable c = s[i]; + + if (c == '\\') + { + if (i + 1 >= s.length) + throw new JSONIopipeException("Invalid escape sequence: lone backslash at end of string"); + + immutable next = s[i + 1]; + + // Allow \', but only when we're using single-quoted strings + if (next == '\'' && currentQuoteChar == '\'') + { + i += 2; + continue; + } + + // Standard JSON escapes + switch (next) + { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + i += 2; + continue; + case 'u': + if (i + 5 >= s.length) + throw new JSONIopipeException("Invalid \\u escape: too short"); + + foreach (j; 2 .. 6) + { + immutable h = s[i + j]; + if (!isHexDigit(h)) + throw new JSONIopipeException("Invalid \\u escape: non-hex digit"); + } + + i += 6; // '\\', 'u', and 4 hex digits + continue; + default: + throw new JSONIopipeException("Invalid escape sequence in string"); + } + } + else + { + // Active quote character must be escaped + if (c == currentQuoteChar) + throw new JSONIopipeException("Unescaped quote character in string"); + + // Bare control characters must not appear; they must be escaped + if (c < 0x20) + throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + + ++i; + } + } + + // In validate mode we output the string unchanged + putStr(s); + } + else static if (mode == StringMode.addEscapes) + { + // Escape any characters that would be invalid in JSON + auto escaped = value.substitute!(jsonEscapeSubstitutions!()).to!string; + + // if we started with a single quote, also escape any remaining ' + if (currentQuoteChar == '\'') + { + import std.array : appender; + auto app = appender!string(); + foreach (char c; escaped) + { + if (c == '\'') + { + app.put('\\'); + app.put('\''); + } + else + { + app.put(c); + } + } + putStr(app.data); + } + else + { + // normal JSON style (double quotes only) + putStr(escaped); + } + } + else static if (mode == StringMode.passThru) + { + // Assume the caller already provided a valid JSON string fragment + putStr(value); + } + } + + void endString() { + if (currentQuoteChar == char.init) + throw new JSONIopipeException("cannot endString without a matching beginString"); + + putStr((¤tQuoteChar)[0 .. 1]); + currentQuoteChar = char.init; + + if (stackLen == 0) { + state = State.End; + putIndent(); + } else + // If it's at the end of an array or an object, + // endArray() and endObject() will set the state appropriately + state = State.Member; + } + + // allow numeric types, or strings that are validated to be JSON numbers + void addNumericData(T)(T value, string formatStr = "%s") { + if (!canAddValue()) + throw new JSONIopipeException("addNumericData not allowed in current state"); + + static if (is(T == string)) { + if (!isValidJSONNumber(value)) { + throw new JSONIopipeException(format("Invalid JSON number: %s", value)); + } + putStr(value); + } else { + static assert(isNumeric!T, "addNumericData requires a numeric type"); + formattedWrite(&putStr, formatStr, value); + } + + if (stackLen == 0) { + state = State.End; + putIndent(); + } else + state = State.Member; + } + + // null, true, false, inf, etc. + void addKeywordValue(KeywordValue value) { + if (!canAddValue()) + throw new JSONIopipeException("addKeywordValue not allowed in current state"); + + putStr(value); + + if (stackLen == 0) { + state = State.End; + putIndent(); + } + else + state = State.Member; + } + + void flushWritten() { + outputChain.release(nWritten); + nWritten = 0; + } + + auto chain() { + return outputChain; + } + +} + +private struct TestChain +{ + char[] buf; + + @property char[] window() + { + return buf; + } + + // Grow buffer when formatter calls ensureElems via free function + size_t extend(size_t elements) + { + import core.memory : GC; + + // extend is called by ensureElems until window.length >= requested + auto oldLen = buf.length; + auto newLen = oldLen + elements; + auto p = cast(char*)GC.malloc(newLen * char.sizeof); + auto newBuf = p[0 .. newLen]; + + if (oldLen) + newBuf[0 .. oldLen] = buf[]; + + buf = newBuf; + return elements; + } + + void release(size_t /*elements*/) + { + // formatter calls release(nWritten); for tests we can ignore it + } +} + +unittest +{ + import iopipe.traits : isIopipe; + import iopipe.bufpipe : ensureElems; + import std.string : strip; + import std.stdio : writeln; + + TestChain chain; + static assert(isIopipe!TestChain); + + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginObject(); + fmt.addMember("a"); + fmt.addNumericData(1); + fmt.endObject(); + fmt.flushWritten(); + + auto s = cast(string)chain.buf.strip; + assert(s == "{\n \"a\": 1\n}"); +} + +unittest +{ + // Simple array: [1,2,3] + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginArray(); + fmt.beginArrayValue(); + fmt.addNumericData(1); + fmt.beginArrayValue(); + fmt.addNumericData(2); + fmt.beginArrayValue(); + fmt.addNumericData(3); + fmt.endArray(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + assert(s == "[\n 1,\n 2,\n 3\n]"); +} + +unittest +{ + // String with escapes and single‑quote behavior + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + // double‑quoted: inner " and \ should be escaped + fmt.beginString('"'); + fmt.addStringData(`He said: "hi" \ test`); + fmt.endString(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + assert(s == `"He said: \"hi\" \\ test"`); + + // reset chain, now test single‑quoted: inner ' must be escaped as \' + chain.buf.length = 0; + auto fmt2 = JSONFormatter!(TestChain)(chain); + + fmt2.beginString('\''); + fmt2.addStringData(`It's ok`); + fmt2.endString(); + fmt2.flushWritten(); + + s = cast(string)chain.buf.strip; + assert(s == `'It\'s ok'`); +} + +unittest +{ + // addEscapes: real newline becomes literal "\n" in JSON + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginString('"'); + fmt.addStringData("line1\nline2"); // contains an actual newline character + fmt.endString(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + // Expect JSON text with a backslash-n escape, not a raw newline + assert(s == `"line1\nline2"`); +} + +unittest +{ + // StringMode.validate: accept only correctly escaped content + import std.exception : assertThrown; + import std.string : strip; + + alias Fmt = JSONFormatter!(TestChain); + alias SM = Fmt.StringMode; + + TestChain chain; + + // 1) Double‑quoted string with valid escapes should pass unchanged + auto fmt = Fmt(chain); + fmt.beginString('"'); + fmt.addStringData!(SM.validate)(`He said: \"hi\" \\ test`); + fmt.endString(); + fmt.flushWritten(); + + auto s = cast(string)chain.buf.strip; + assert(s == `"He said: \"hi\" \\ test"`); + + // 2) Invalid escape ("\q") should throw in validate mode + chain.buf.length = 0; + auto fmt2 = Fmt(chain); + fmt2.beginString('"'); + assertThrown!JSONIopipeException(fmt2.addStringData!(SM.validate)(`He said: \q`)); + + // 3) Lone backslash at end should throw in validate mode + chain.buf.length = 0; + auto fmt3 = Fmt(chain); + fmt3.beginString('"'); + assertThrown!JSONIopipeException(fmt3.addStringData!(SM.validate)(`oops \`)); + + // 4) Unescaped quote should throw in validate mode + chain.buf.length = 0; + auto fmt4 = Fmt(chain); + fmt4.beginString('"'); + assertThrown!JSONIopipeException(fmt4.addStringData!(SM.validate)(`He said: "hi"`)); + + // 5) Single‑quoted validate: \' is allowed, bare ' is not + chain.buf.length = 0; + auto fmt5 = Fmt(chain); + fmt5.beginString('\''); + fmt5.addStringData!(SM.validate)(`It\'s ok`); + fmt5.endString(); + fmt5.flushWritten(); + + s = cast(string)chain.buf.strip; + assert(s == `'It\'s ok'`); + + chain.buf.length = 0; + auto fmt6 = Fmt(chain); + fmt6.beginString('\''); + assertThrown!JSONIopipeException(fmt6.addStringData!(SM.validate)(`It's bad`)); +} + +unittest +{ + // StringMode.validate: cover all standard JSON escapes and \u forms + import std.exception : assertThrown; + + alias Fmt = JSONFormatter!(TestChain); + alias SM = Fmt.StringMode; + + TestChain chain; + + // 1) All standard single-character escapes should be accepted + auto fmt = Fmt(chain); + fmt.beginString('"'); + fmt.addStringData!(SM.validate)(`\" \\ \/ \b \f \n \r \t`); + fmt.endString(); + fmt.flushWritten(); + + // 2) Valid \u escape should be accepted + chain.buf.length = 0; + auto fmt2 = Fmt(chain); + fmt2.beginString('"'); + fmt2.addStringData!(SM.validate)(`\u00AF`); + fmt2.endString(); + fmt2.flushWritten(); + + // 3) Too-short \u escape should throw + chain.buf.length = 0; + auto fmt3 = Fmt(chain); + fmt3.beginString('"'); + assertThrown!JSONIopipeException(fmt3.addStringData!(SM.validate)(`\u12`)); + + // 4) \u escape with non-hex digits should throw + chain.buf.length = 0; + auto fmt4 = Fmt(chain); + fmt4.beginString('"'); + assertThrown!JSONIopipeException(fmt4.addStringData!(SM.validate)(`\u12xz`)); + + // 5) Bare control character (< 0x20) should throw + chain.buf.length = 0; + auto fmt5 = Fmt(chain); + fmt5.beginString('"'); + string bad = "ok" ~ "\x01"; // embed a real control char 0x01 + assertThrown!JSONIopipeException(fmt5.addStringData!(SM.validate)(bad)); +} + +unittest +{ + // Keywords: null/true/false + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginArray(); + fmt.beginArrayValue(); + fmt.addKeywordValue(KeywordValue.Null); + fmt.beginArrayValue(); + fmt.addKeywordValue(KeywordValue.True); + fmt.beginArrayValue(); + fmt.addKeywordValue(KeywordValue.False); + fmt.endArray(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + assert(s == "[\n null,\n true,\n false\n]"); +} + +unittest +{ + // ERROR: endArray without beginArray + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.endArray()); +} + +unittest +{ + // ERROR: addMember when not inside object + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.addMember("a")); +} + +unittest +{ + // ERROR: add key string directily without in addMember + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + fmt.beginObject(); + + assertThrown!JSONIopipeException(fmt.beginString()); +} + +unittest +{ + // ERROR: addArrayValue when not inside array + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.beginArrayValue()); +} + +unittest +{ + // ERROR: nested beginString + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginString(); + assertThrown!JSONIopipeException(fmt.beginString()); +} + +unittest +{ + // ERROR: addStringData without beginString + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.addStringData("oops")); +} + +unittest +{ + // ERROR: endString without beginString + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.endString()); +} + +unittest +{ + // ERROR: invalid JSON number string + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.addNumericData("01")); +} \ No newline at end of file diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index 285cc46..c352c60 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -1352,25 +1352,9 @@ void deserializeArray(T, JT, Policy)( // Parse array elements size_t elementCount = 0; - while(true) { + while(tokenizer.peekSkipComma() != JSONToken.ArrayEnd) { policy.onArrayElement(tokenizer, item, elementCount, context); elementCount++; - - if (tokenizer.peekSignificant() == JSONToken.ArrayEnd) { - // If we hit the end of the array, break - break; - } - - // verify and consume the comma - jsonItem = tokenizer.nextSignificant() - .jsonExpect(JSONToken.Comma, "Parsing " ~ T.stringof); - - static if (tokenizer.config.JSON5) - { - if (tokenizer.peekSignificant() == JSONToken.ArrayEnd) - break; - } - } // verify we got an end array element @@ -1904,7 +1888,7 @@ unittest * Result: * AliasSeq!("", "", "", "", ...); */ -private template jsonEscapeSubstitutions() +package template jsonEscapeSubstitutions() { import std.algorithm.iteration; import std.range;