From e3f2528c5a621dc2a4c45082f6a514bca646ed22 Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Tue, 23 Sep 2025 12:01:07 -0600 Subject: [PATCH 1/9] Formatter Serialize --- examples/formatjson/formatjson.d | 62 +++++++- source/iopipe/json/serialize.d | 250 +++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+), 1 deletion(-) diff --git a/examples/formatjson/formatjson.d b/examples/formatjson/formatjson.d index 52deb95..9d97a59 100644 --- a/examples/formatjson/formatjson.d +++ b/examples/formatjson/formatjson.d @@ -7,6 +7,8 @@ import iopipe.textpipe; import iopipe.valve; import iopipe.json.parser; import iopipe.buffer; +import iopipe.json.serialize; +import iopipe.json.dom; void main(string[] args) { @@ -20,7 +22,7 @@ void main(string[] args) auto outputter = bufd!(char).push!(c => c .encodeText!(UTFType.UTF8) .outputPipe(File(stdout).refCounted)); - + /* int spacing; enum indent = 4; void putIndent() @@ -92,4 +94,62 @@ void main(string[] args) item = parser.next; } putStr("\n"); + */ + + JSONValue!string jsonData; + deserialize(parser, jsonData); + auto formatter = JSONFormatter!(typeof(outputter))(outputter); + + serializeWithFormatter(formatter, jsonData); + formatter.addWhitespace("\n"); + formatter.flushWritten(); +} + +void serializeWithFormatter(Formatter, T)(ref Formatter formatter, T val) +{ + static if (is(T == JSONValue!string)) + { + with(JSONType) final switch(val.type) + { + case Obj: + formatter.beginObject(); + bool first = true; + foreach(key, value; val.object) + { + formatter.addMember(key); + serializeWithFormatter(formatter, value); + } + formatter.endObject(); + break; + case Array: + formatter.beginArray(); + foreach(item; val.array) + { + formatter.addMember(""); // dummy key for array element + serializeWithFormatter(formatter, item); + } + formatter.endArray(); + break; + case String: + formatter.beginString(); + formatter.addStringData(val.str); + formatter.endString(); + break; + case Integer: + formatter.addNumericData(val.integer); + break; + case Floating: + formatter.addNumericData(val.floating); + break; + case Bool: + if(val.boolean) + formatter.addKeywordValue(KeywordValue.True); + else + formatter.addKeywordValue(KeywordValue.False); + break; + case Null: + formatter.addKeywordValue(KeywordValue.Null); + break; + } + } } diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index 285cc46..8fd2b4d 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -31,6 +31,225 @@ import std.typecons : Nullable; import std.conv; import std.format; +struct MemberOptions { + char quoteChar = '"'; // Use " or ' for quotes + bool spaceBeforeColon = false; // "key" : vs "key": + bool spaceAfterColon = true; // "key": vs "key" : + bool escapeKey = true; // Whether to escape special chars in key +} + +enum KeywordValue { + Null, + True, + False, + // JSON5 extensions + Infinity, + NegativeInfinity, + NaN +} + +struct JSONFormatter(Chain) { +private: + Chain* outputChain; + int spacing = 0; + enum indent = 4; + + bool[] isFirstInObj; + bool[] isFirstInArray; + size_t nWritten = 0; + + void putIndent() { + outputChain.ensureElems(nWritten + indent * spacing + 1); + outputChain.window[nWritten] = '\n'; + outputChain.window[nWritten + 1 .. nWritten + 1 + indent * spacing] = ' '; + nWritten += indent * spacing + 1; + } + + void putStr(const(char)[] s) { + outputChain.ensureElems(nWritten + s.length); + outputChain.window[nWritten .. nWritten + s.length] = s; + nWritten += s.length; + } + + private bool isValidJSONNumber(string str) { + import std.regex; + static auto numberRegex = regex(r"^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$"); + return !str.matchFirst(numberRegex).empty; + } + +public: + this(ref Chain chain) { + outputChain = &chain; + } + + void beginObject() { + isFirstInObj ~= true; + ++spacing; + putStr("{"); + putIndent(); + } + + void endObject() { + isFirstInObj = isFirstInObj[0..$-1]; + --spacing; + putIndent(); + putStr("}"); + } + + void beginArray() { + isFirstInArray ~= true; + ++spacing; + putStr("["); + putIndent(); + } + + void endArray() { + isFirstInArray = isFirstInArray[0..$-1]; + --spacing; + putIndent(); + putStr("]"); + } + + void addMember(T)(T key, MemberOptions options = MemberOptions.init) { + + if (key.length == 0 ) { + // use for array elements + if (!isFirstInArray[$-1]) { + putStr(", "); + putIndent(); + } + isFirstInArray[$-1] = false; + return; + } + + if (!isFirstInObj[$-1]) { + putStr(","); + putIndent(); + } + + static if (is(typeof(key) == string)) { + if (options.escapeKey) { + beginString(options.quoteChar); + addStringData(key); + endString(options.quoteChar); + } else { + beginString(options.quoteChar); + addStringData!(false, false)(key); + endString(options.quoteChar); + } + } else { + putStr(to!string(key)); + } + if (options.spaceBeforeColon) { + putStr(" "); + } + putStr(":"); + if (options.spaceAfterColon) { + putStr(" "); + } + + isFirstInObj[$-1] = false; + } + + void beginString(char quoteChar = '"') { + putStr([quoteChar]); + } + + // will automatically escape string data as needed. + void addStringData(bool validate = true, bool addEscapes = true, T)(T value) { + static if (validate) { + // Validate that the string doesn't contain invalid characters + foreach(char c; value) { + if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') { + throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + } + } + } + + static if (addEscapes) { + import std.algorithm.iteration: substitute; + import std.conv: to; + auto escaped = value.substitute!(jsonEscapeSubstitutions!()).to!string; + putStr(escaped); + } else { + putStr(value); + } + } + + + void endString(char quoteChar = '"') { + putStr([quoteChar]); + } + + // allow numeric types, or strings that are validated to be JSON numbers + void addNumericData(T)(T value, string formatStr = "%s") { + static if (is(T == string)) { + if (!isValidJSONNumber(value)) { + throw new JSONIopipeException(format("Invalid JSON number: %s", value)); + } + putStr(value); + } else { + import std.format : format; + putStr(format(formatStr, value)); + } + } + + // null, true, false, inf, etc. + void addKeywordValue(KeywordValue value) { + final switch (value) { + case KeywordValue.Null: + putStr("null"); + break; + case KeywordValue.True: + putStr("true"); + break; + case KeywordValue.False: + putStr("false"); + break; + case KeywordValue.Infinity: + putStr("Infinity"); + break; + case KeywordValue.NegativeInfinity: + putStr("-Infinity"); + break; + case KeywordValue.NaN: + putStr("NaN"); + break; + } + } + + void addWhitespace(T)(T data) { + foreach(char c; data) { + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + throw new JSONIopipeException(format("Invalid whitespace character: \\u%04X", cast(int)c)); + } + } + putStr(data); + } + + // // add a comment (JSON5 only), must be a complete comment (validated) + void addComment(T)(T commentData) { + static if (is(T == string)) { + import std.algorithm.searching : startsWith, endsWith; + if (!commentData.startsWith("//") && !commentData.startsWith("/*") && !commentData.endsWith("*/")) { + throw new JSONIopipeException(format("Invalid comment format: %s", commentData)); + } + } + putStr(commentData); + } + + void flushWritten() { + outputChain.release(nWritten); + nWritten = 0; + } + + auto chain() { + return outputChain; + } + +} + + struct DefaultDeserializationPolicy(bool caseInsensitive = false) { ReleasePolicy relPol = ReleasePolicy.afterMembers; // default policy int maxDepthAvailable = 64; @@ -734,6 +953,31 @@ void deserializeImpl(P, T, JT)(ref P policy, ref JT tokenizer, ref T item) if (i } } +unittest { + // Test emptyObject first + auto jsonStr1 = `{"emptyObject": {}}`; + auto jv1 = deserialize!(JSONValue!string)(jsonStr1); + + assert(jv1.type == JSONType.Obj); + assert("emptyObject" in jv1.object); + + auto emptyObj = jv1.object["emptyObject"]; + assert(emptyObj.type == JSONType.Obj); + assert(emptyObj.object.length == 0); + + auto jsonStr2 = `{"emptyArray": []}`; + auto jv2 = deserialize!(JSONValue!string)(jsonStr2); + + assert(jv2.type == JSONType.Obj); + assert("emptyArray" in jv2.object); + + auto emptyArr = jv2.object["emptyArray"]; + assert(emptyArr.type == JSONType.Array); + assert(emptyArr.array.length == 0); +} + + + void deserializeAllMembers(T, JT)(ref JT tokenizer, ref T item, ReleasePolicy relPol) { // expect an object in JSON. We want to deserialize the JSON data @@ -1353,6 +1597,12 @@ void deserializeArray(T, JT, Policy)( // Parse array elements size_t elementCount = 0; while(true) { + + if (tokenizer.peekSignificant() == JSONToken.ArrayEnd) { + // Handle empty array case + break; + } + policy.onArrayElement(tokenizer, item, elementCount, context); elementCount++; From d209378a11be085ec1643409e4b40c8995c170b6 Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Thu, 25 Sep 2025 19:08:47 -0600 Subject: [PATCH 2/9] polish the Formatter --- source/iopipe/json/serialize.d | 101 +++++++++++++++++---------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index 8fd2b4d..ec0714d 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -33,29 +33,36 @@ import std.format; struct MemberOptions { char quoteChar = '"'; // Use " or ' for quotes - bool spaceBeforeColon = false; // "key" : vs "key": - bool spaceAfterColon = true; // "key": vs "key" : - bool escapeKey = true; // Whether to escape special chars in key + bool escapeKey = true; // Whether to escape special chars in key + + enum ColonSpacing { + none, // "key":"value" + after, // "key": "value" + both, // "key" : "value" + } + + ColonSpacing colonSpacing = ColonSpacing.after; // Default spacing } -enum KeywordValue { - Null, - True, - False, +enum KeywordValue : string { + Null = "null", + True = "true", + False = "false", // JSON5 extensions - Infinity, - NegativeInfinity, - NaN + Infinity = "Infinity", + NegativeInfinity = "-Infinity", + NaN = "NaN" } struct JSONFormatter(Chain) { private: Chain* outputChain; int spacing = 0; - enum indent = 4; + int indent = 4; + MemberOptions memberOptions; - bool[] isFirstInObj; - bool[] isFirstInArray; + bool isFirstInObj; + bool isFirstInArray; size_t nWritten = 0; void putIndent() { @@ -82,47 +89,55 @@ public: outputChain = &chain; } + this(ref Chain chain, int spacing, int indent) { + outputChain = &chain; + this.spacing = spacing; + this.indent = indent; + } + void beginObject() { - isFirstInObj ~= true; + isFirstInObj = true; ++spacing; putStr("{"); putIndent(); } void endObject() { - isFirstInObj = isFirstInObj[0..$-1]; --spacing; putIndent(); putStr("}"); } void beginArray() { - isFirstInArray ~= true; + isFirstInArray = true; ++spacing; putStr("["); putIndent(); } void endArray() { - isFirstInArray = isFirstInArray[0..$-1]; --spacing; putIndent(); putStr("]"); } - void addMember(T)(T key, MemberOptions options = MemberOptions.init) { + void addMember(T)(T key) { + addMember(key, memberOptions); + } + + void addMember(T)(T key, MemberOptions options) { if (key.length == 0 ) { // use for array elements - if (!isFirstInArray[$-1]) { + if (!isFirstInArray) { putStr(", "); putIndent(); } - isFirstInArray[$-1] = false; + isFirstInArray = false; return; } - if (!isFirstInObj[$-1]) { + if (!isFirstInObj) { putStr(","); putIndent(); } @@ -140,15 +155,20 @@ public: } else { putStr(to!string(key)); } - if (options.spaceBeforeColon) { - putStr(" "); - } - putStr(":"); - if (options.spaceAfterColon) { - putStr(" "); + // Handle colon spacing based on enum value + final switch (options.colonSpacing) { + case MemberOptions.ColonSpacing.none: + putStr(":"); // "key":"value" + break; + case MemberOptions.ColonSpacing.after: + putStr(": "); // "key": "value" + break; + case MemberOptions.ColonSpacing.both: + putStr(" : "); // "key" : "value" + break; } - isFirstInObj[$-1] = false; + isFirstInObj = false; } void beginString(char quoteChar = '"') { @@ -189,33 +209,14 @@ public: } putStr(value); } else { - import std.format : format; - putStr(format(formatStr, value)); + import std.format : formattedWrite; + formattedWrite(&putStr, formatStr, value); } } // null, true, false, inf, etc. void addKeywordValue(KeywordValue value) { - final switch (value) { - case KeywordValue.Null: - putStr("null"); - break; - case KeywordValue.True: - putStr("true"); - break; - case KeywordValue.False: - putStr("false"); - break; - case KeywordValue.Infinity: - putStr("Infinity"); - break; - case KeywordValue.NegativeInfinity: - putStr("-Infinity"); - break; - case KeywordValue.NaN: - putStr("NaN"); - break; - } + putStr(value); } void addWhitespace(T)(T data) { From 6a3173d574b724d8980436db8ce045002386bcea Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Fri, 26 Sep 2025 12:54:27 -0600 Subject: [PATCH 3/9] delete unrelated modification --- source/iopipe/json/serialize.d | 43 +++++----------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index ec0714d..be34d07 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -178,13 +178,13 @@ public: // will automatically escape string data as needed. void addStringData(bool validate = true, bool addEscapes = true, T)(T value) { static if (validate) { - // Validate that the string doesn't contain invalid characters - foreach(char c; value) { - if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') { - throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + // Validate that the string doesn't contain invalid characters + foreach(char c; value) { + if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') { + throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + } } } - } static if (addEscapes) { import std.algorithm.iteration: substitute; @@ -954,31 +954,6 @@ void deserializeImpl(P, T, JT)(ref P policy, ref JT tokenizer, ref T item) if (i } } -unittest { - // Test emptyObject first - auto jsonStr1 = `{"emptyObject": {}}`; - auto jv1 = deserialize!(JSONValue!string)(jsonStr1); - - assert(jv1.type == JSONType.Obj); - assert("emptyObject" in jv1.object); - - auto emptyObj = jv1.object["emptyObject"]; - assert(emptyObj.type == JSONType.Obj); - assert(emptyObj.object.length == 0); - - auto jsonStr2 = `{"emptyArray": []}`; - auto jv2 = deserialize!(JSONValue!string)(jsonStr2); - - assert(jv2.type == JSONType.Obj); - assert("emptyArray" in jv2.object); - - auto emptyArr = jv2.object["emptyArray"]; - assert(emptyArr.type == JSONType.Array); - assert(emptyArr.array.length == 0); -} - - - void deserializeAllMembers(T, JT)(ref JT tokenizer, ref T item, ReleasePolicy relPol) { // expect an object in JSON. We want to deserialize the JSON data @@ -1597,13 +1572,7 @@ void deserializeArray(T, JT, Policy)( // Parse array elements size_t elementCount = 0; - while(true) { - - if (tokenizer.peekSignificant() == JSONToken.ArrayEnd) { - // Handle empty array case - break; - } - + while(true) { policy.onArrayElement(tokenizer, item, elementCount, context); elementCount++; From e668d25a2fbe9c0b4c8113110af14541a2313695 Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Mon, 24 Nov 2025 18:36:21 -0700 Subject: [PATCH 4/9] Refine comma, string and numeric handling --- examples/formatjson/formatjson.d | 2 +- source/iopipe/json/serialize.d | 87 ++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/examples/formatjson/formatjson.d b/examples/formatjson/formatjson.d index 9d97a59..0c6f8e2 100644 --- a/examples/formatjson/formatjson.d +++ b/examples/formatjson/formatjson.d @@ -125,7 +125,7 @@ void serializeWithFormatter(Formatter, T)(ref Formatter formatter, T val) formatter.beginArray(); foreach(item; val.array) { - formatter.addMember(""); // dummy key for array element + formatter.beginArrayValue(); serializeWithFormatter(formatter, item); } formatter.endArray(); diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index be34d07..c821356 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -65,6 +65,8 @@ private: bool isFirstInArray; size_t nWritten = 0; + char currentQuoteChar = '"'; + void putIndent() { outputChain.ensureElems(nWritten + indent * spacing + 1); outputChain.window[nWritten] = '\n'; @@ -78,12 +80,32 @@ private: nWritten += s.length; } - private bool isValidJSONNumber(string str) { + bool isValidJSONNumber(string str) { import std.regex; static auto numberRegex = regex(r"^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$"); return !str.matchFirst(numberRegex).empty; } + void startObjectMember() + { + if (!isFirstInObj) { + putStr(","); + putIndent(); + } else { + isFirstInObj = false; + } + } + + void startArrayValue() + { + if (!isFirstInArray) { + putStr(","); + putIndent(); + } else { + isFirstInArray = false; + } + } + public: this(ref Chain chain) { outputChain = &chain; @@ -127,30 +149,17 @@ public: void addMember(T)(T key, MemberOptions options) { - if (key.length == 0 ) { - // use for array elements - if (!isFirstInArray) { - putStr(", "); - putIndent(); - } - isFirstInArray = false; - return; - } - - if (!isFirstInObj) { - putStr(","); - putIndent(); - } + startObjectMember(); static if (is(typeof(key) == string)) { if (options.escapeKey) { beginString(options.quoteChar); addStringData(key); - endString(options.quoteChar); + endString(); } else { beginString(options.quoteChar); addStringData!(false, false)(key); - endString(options.quoteChar); + endString(); } } else { putStr(to!string(key)); @@ -171,8 +180,17 @@ public: isFirstInObj = false; } + // expose startArrayValue to callers that emit arrays + void beginArrayValue() { + startArrayValue(); + } + void beginString(char quoteChar = '"') { - putStr([quoteChar]); + if (quoteChar != '"' && quoteChar != '\'') + throw new JSONIopipeException("Invalid quote character"); + + currentQuoteChar = quoteChar; + putStr(("eChar)[0 .. 1]); } // will automatically escape string data as needed. @@ -197,8 +215,8 @@ public: } - void endString(char quoteChar = '"') { - putStr([quoteChar]); + void endString() { + putStr((¤tQuoteChar)[0 .. 1]); } // allow numeric types, or strings that are validated to be JSON numbers @@ -209,6 +227,7 @@ public: } putStr(value); } else { + static assert(isNumeric!T, "addNumericData requires a numeric type"); import std.format : formattedWrite; formattedWrite(&putStr, formatStr, value); } @@ -1560,6 +1579,16 @@ void deserializeImpl(T, JT, Policy)( } } +auto peekSkipComma(JT)(ref JT tokenizer) +{ + auto token = tokenizer.peekSignificant(); + if(token != JSONToken.Comma) + return token; + // consume the comma + cast(void)tokenizer.nextSignificant; + return tokenizer.peekSignificant(); +} + void deserializeArray(T, JT, Policy)( ref Policy policy, ref JT tokenizer, @@ -1572,25 +1601,9 @@ void deserializeArray(T, JT, Policy)( // Parse array elements size_t elementCount = 0; - while(true) { + while(tokenizer.peekSkipComma() != JSONToken.ArrayEnd) { policy.onArrayElement(tokenizer, item, elementCount, context); elementCount++; - - if (tokenizer.peekSignificant() == JSONToken.ArrayEnd) { - // If we hit the end of the array, break - break; - } - - // verify and consume the comma - jsonItem = tokenizer.nextSignificant() - .jsonExpect(JSONToken.Comma, "Parsing " ~ T.stringof); - - static if (tokenizer.config.JSON5) - { - if (tokenizer.peekSignificant() == JSONToken.ArrayEnd) - break; - } - } // verify we got an end array element From 71aab951556e1ebc07b2476e59e10a76688045f2 Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Mon, 24 Nov 2025 18:57:22 -0700 Subject: [PATCH 5/9] Remove duplicate peekSkipComma, use master version --- source/iopipe/json/serialize.d | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index c821356..dfa37bb 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -1579,16 +1579,6 @@ void deserializeImpl(T, JT, Policy)( } } -auto peekSkipComma(JT)(ref JT tokenizer) -{ - auto token = tokenizer.peekSignificant(); - if(token != JSONToken.Comma) - return token; - // consume the comma - cast(void)tokenizer.nextSignificant; - return tokenizer.peekSignificant(); -} - void deserializeArray(T, JT, Policy)( ref Policy policy, ref JT tokenizer, From b28db949579f87a6e9da0d610fef681c2c02b4ff Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Wed, 26 Nov 2025 08:43:12 -0700 Subject: [PATCH 6/9] split formatter from serialize and add state in formatter --- examples/formatjson/formatjson.d | 2 +- source/iopipe/json/formatter.d | 466 +++++++++++++++++++++++++++++++ source/iopipe/json/serialize.d | 243 +--------------- 3 files changed, 475 insertions(+), 236 deletions(-) create mode 100644 source/iopipe/json/formatter.d diff --git a/examples/formatjson/formatjson.d b/examples/formatjson/formatjson.d index 0c6f8e2..12e1f9f 100644 --- a/examples/formatjson/formatjson.d +++ b/examples/formatjson/formatjson.d @@ -9,6 +9,7 @@ import iopipe.json.parser; import iopipe.buffer; import iopipe.json.serialize; import iopipe.json.dom; +import iopipe.json.formatter; void main(string[] args) { @@ -113,7 +114,6 @@ void serializeWithFormatter(Formatter, T)(ref Formatter formatter, T val) { case Obj: formatter.beginObject(); - bool first = true; foreach(key, value; val.object) { formatter.addMember(key); diff --git a/source/iopipe/json/formatter.d b/source/iopipe/json/formatter.d new file mode 100644 index 0000000..af5afbb --- /dev/null +++ b/source/iopipe/json/formatter.d @@ -0,0 +1,466 @@ +module iopipe.json.formatter; + +public import iopipe.json.common; + +import iopipe.traits : isIopipe, WindowType; +import iopipe.bufpipe; + +import std.bitmanip : BitArray; +import std.exception : enforce; +import std.traits : isNumeric; +import std.conv : to; +import std.regex; +import std.algorithm.iteration : substitute; +import std.algorithm.searching : startsWith, endsWith; +import std.format; + + +struct MemberOptions { + char quoteChar = '"'; // Use " or ' for quotes + bool escapeKey = true; // Whether to escape special chars in key + + enum ColonSpacing { + none, // "key":"value" + after, // "key": "value" + both, // "key" : "value" + } + + ColonSpacing colonSpacing = ColonSpacing.after; // Default spacing +} + +enum KeywordValue : string { + Null = "null", + True = "true", + False = "false", + // JSON5 extensions + Infinity = "Infinity", + NegativeInfinity = "-Infinity", + NaN = "NaN" +} + +struct JSONFormatter(Chain) +if (isIopipe!Chain) +{ +private: + + Chain* outputChain; + int spacing = 0; + int indent = 4; + MemberOptions memberOptions; + + size_t nWritten = 0; + char currentQuoteChar = '"'; + + private enum State : ubyte + { + Begin, // next item should be either an Object or Array or String + First, // Just started a new object or array. + Member, // Expect next member (name for object, value for array_ + Value, // Expect value + End // there shouldn't be any more items + } + + // 0 = array, 1 = object (same convention as parser) + BitArray stack; + size_t stackLen; + State state = State.Begin; + + bool inObj() @property + { + return stackLen == 0 ? false : stack[stackLen - 1]; + } + + void pushContainer(bool isObj) + { + if (stackLen == stack.length) + stack ~= isObj; + else + stack[stackLen] = isObj; + ++stackLen; + } + + void popContainer() + { + state = (--stackLen == 0) ? State.End : State.Member; + } + + void putIndent() { + outputChain.ensureElems(nWritten + indent * spacing + 1); + outputChain.window[nWritten] = '\n'; + outputChain.window[nWritten + 1 .. nWritten + 1 + indent * spacing] = ' '; + nWritten += indent * spacing + 1; + } + + void putStr(const(char)[] s) { + outputChain.ensureElems(nWritten + s.length); + outputChain.window[nWritten .. nWritten + s.length] = s; + nWritten += s.length; + } + + bool isValidJSONNumber(string str) { + static auto numberRegex = regex(r"^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$"); + return !str.matchFirst(numberRegex).empty; + } + + void startObjectMember() + { + enforce(inObj(), "startObjectMember only valid inside object"); + + final switch (state) with (State) + { + case First: + // first member: no comma + state = Member; + break; + case Member: + // subsequent member: comma + indent + putStr(","); + putIndent(); + // stay in Member + break; + case Begin, Value, End: + throw new JSONIopipeException( + "startObjectMember not allowed in current state"); + } + } + + void startArrayValue() + { + enforce(!inObj(), "startArrayValue only valid inside array"); + + final switch (state) with (State) + { + case First: + // first element: no comma + state = Member; + break; + case Member: + // subsequent element: comma + indent + putStr(","); + putIndent(); + // stay in Member + break; + case Begin, Value, End: + throw new JSONIopipeException( + "startArrayValue not allowed in current state"); + } + } + + private void endStringRaw() { + putStr((¤tQuoteChar)[0 .. 1]); + // no state change + } + +public: + this(ref Chain chain) { + outputChain = &chain; + } + + this(ref Chain chain, int spacing, int indent) { + outputChain = &chain; + this.spacing = spacing; + this.indent = indent; + } + + void beginObject() { + final switch (state) with (State) + { + case Begin, Member, Value: + break; + case First, End: + throw new JSONIopipeException( + "beginObject not allowed in current state"); + } + + pushContainer(true); // object + state = State.First; // just started + + ++spacing; + putStr("{"); + putIndent(); + } + + void endObject() { + enforce(inObj(), "endObject outside object"); + + --spacing; + putIndent(); + putStr("}"); + + popContainer(); // State.End or State.Comma + } + + void beginArray() { + final switch (state) with (State) + { + case Begin, Member, Value: + break; + case First, End: + throw new JSONIopipeException( + "beginArray not allowed in current state"); + } + + pushContainer(false); // array + state = State.First; + + ++spacing; + putStr("["); + putIndent(); + } + + void endArray() { + enforce(!inObj(), "endArray outside array"); + + --spacing; + putIndent(); + putStr("]"); + + popContainer(); + } + + // Public hook for arrays: call before each element + void beginArrayValue() + { + startArrayValue(); + } + + void addMember(T)(T key) { + addMember(key, memberOptions); + } + + void addMember(T)(T key, MemberOptions options) { + + enforce(inObj(), "addMember only valid inside object"); + + final switch (state) with (State) + { + case First, Member: + break; + case Begin, Value, End: + throw new JSONIopipeException( + "addMember not allowed in current state"); + } + + startObjectMember(); + + static if (is(typeof(key) == string)) + { + if (options.escapeKey) + { + beginString(options.quoteChar); + addStringData(key); + endStringRaw(); + } + else + { + beginString(options.quoteChar); + addStringData!(false, false)(key); + endStringRaw(); + } + } + else + { + putStr(to!string(key)); + } + + final switch (options.colonSpacing) + { + case MemberOptions.ColonSpacing.none: + putStr(":"); + break; + case MemberOptions.ColonSpacing.after: + putStr(": "); + break; + case MemberOptions.ColonSpacing.both: + putStr(" : "); + break; + } + + state = State.Value; + } + + void beginString(char quoteChar = '"') { + // value allowed at: Begin (root), Member (array element), Value (object value) + final switch (state) with (State) + { + case Begin, Member, Value: + break; + case First, End: + throw new JSONIopipeException( + "beginString not allowed in current state"); + } + + if (quoteChar != '"' && quoteChar != '\'') + throw new JSONIopipeException("Invalid quote character"); + + currentQuoteChar = quoteChar; + putStr(("eChar)[0 .. 1]); + } + + // will automatically escape string data as needed. + void addStringData(bool validate = true, bool addEscapes = true, T)(T value) { + static if (validate) { + // Validate that the string doesn't contain invalid characters + foreach(char c; value) { + if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') { + throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + } + } + } + + static if (addEscapes) { + auto escaped = value.substitute!(jsonEscapeSubstitutions!()).to!string; + putStr(escaped); + } else { + putStr(value); + } + } + + + void endString() { + putStr((¤tQuoteChar)[0 .. 1]); + + if (stackLen == 0) + state = State.End; + else + // If it's at the end of an array or an object, + // endArray() and endObject() will set the state appropriately + state = State.Member; + } + + // allow numeric types, or strings that are validated to be JSON numbers + void addNumericData(T)(T value, string formatStr = "%s") { + + final switch (state) with (State) + { + case Begin, Member, Value: + break; + case First, End: + throw new JSONIopipeException( + "addNumericData not allowed in current state"); + } + + static if (is(T == string)) { + if (!isValidJSONNumber(value)) { + throw new JSONIopipeException(format("Invalid JSON number: %s", value)); + } + putStr(value); + } else { + static assert(isNumeric!T, "addNumericData requires a numeric type"); + formattedWrite(&putStr, formatStr, value); + } + + if (stackLen == 0) + state = State.End; + else + state = State.Member; + } + + // null, true, false, inf, etc. + void addKeywordValue(KeywordValue value) { + final switch (state) with (State) + { + case Begin, Member, Value: + break; + case First, End: + throw new JSONIopipeException( + "addKeywordValue not allowed in current state"); + } + + putStr(value); + + if (stackLen == 0) + state = State.End; + else + state = State.Member; + } + + void addWhitespace(T)(T data) { + foreach(char c; data) { + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + throw new JSONIopipeException(format("Invalid whitespace character: \\u%04X", cast(int)c)); + } + } + putStr(data); + } + + // // add a comment (JSON5 only), must be a complete comment (validated) + void addComment(T)(T commentData) { + static if (is(T == string)) { + if (!commentData.startsWith("//") && !commentData.startsWith("/*") && !commentData.endsWith("*/")) { + throw new JSONIopipeException(format("Invalid comment format: %s", commentData)); + } + } + putStr(commentData); + } + + void flushWritten() { + outputChain.release(nWritten); + nWritten = 0; + } + + auto chain() { + return outputChain; + } + +} + +/** Eponymous template that generates an argument list for std.algorithm.substitute to correctly escape json strings + * Result: + * AliasSeq!("", "", "", "", ...); + * + * copied from serialize.d + */ +private template jsonEscapeSubstitutions() +{ + import std.algorithm.iteration; + import std.range; + import std.ascii: ControlChar; + + // Control characters [0,31 aka 0x1f] + 2 special characters '"' and '\' + private enum charactersToEscape = chain(only('\"', '\\'), iota(0x1f + 1)); + + private struct JsonEscapeMapping { + char chr; + string escapeSequence; + } + + /* Special characters we have to (in case of '"' and '\') per the spec or want to escape seperately for readability + * Everything else gets converted to the "\uxxxx" unicode escape + */ + private enum JsonEscapeMapping[7] JSON_ESCAPES = [ + {'\"', `\"`}, + {'\\', `\\`}, + {ControlChar.bs, `\b`}, + {ControlChar.lf, `\n`}, + {ControlChar.cr, `\r`}, + {ControlChar.tab, `\t`}, + {ControlChar.ff, `\f`}, + ]; + + private JsonEscapeMapping escapeSingleChar(int c) + { + switch(c) + { + static foreach(e; JSON_ESCAPES) + { + case e.chr: + return e; + } + default: + return JsonEscapeMapping(cast(char)c, format!`\u%04x`(c)); + } + } + + // Convert struct to AliasSeq with correct types so it can be used as parameters of a function + private template UnpackStruct(JsonEscapeMapping jem) + { + // Must convert char to string for use with substitute + enum string staticCast(char c) = c.to!string; + enum string staticCast(string s) = s; + alias UnpackStruct = staticMap!(staticCast, jem.tupleof); + } + + import std.meta; + private alias jsonEscapeSubstitutions = staticMap!(UnpackStruct, aliasSeqOf!(charactersToEscape.map!escapeSingleChar)); +} \ No newline at end of file diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index dfa37bb..6f2acf3 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -31,242 +31,15 @@ import std.typecons : Nullable; import std.conv; import std.format; -struct MemberOptions { - char quoteChar = '"'; // Use " or ' for quotes - bool escapeKey = true; // Whether to escape special chars in key - - enum ColonSpacing { - none, // "key":"value" - after, // "key": "value" - both, // "key" : "value" - } - - ColonSpacing colonSpacing = ColonSpacing.after; // Default spacing -} - -enum KeywordValue : string { - Null = "null", - True = "true", - False = "false", - // JSON5 extensions - Infinity = "Infinity", - NegativeInfinity = "-Infinity", - NaN = "NaN" -} - -struct JSONFormatter(Chain) { -private: - Chain* outputChain; - int spacing = 0; - int indent = 4; - MemberOptions memberOptions; - - bool isFirstInObj; - bool isFirstInArray; - size_t nWritten = 0; - - char currentQuoteChar = '"'; - - void putIndent() { - outputChain.ensureElems(nWritten + indent * spacing + 1); - outputChain.window[nWritten] = '\n'; - outputChain.window[nWritten + 1 .. nWritten + 1 + indent * spacing] = ' '; - nWritten += indent * spacing + 1; - } - - void putStr(const(char)[] s) { - outputChain.ensureElems(nWritten + s.length); - outputChain.window[nWritten .. nWritten + s.length] = s; - nWritten += s.length; - } - - bool isValidJSONNumber(string str) { - import std.regex; - static auto numberRegex = regex(r"^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$"); - return !str.matchFirst(numberRegex).empty; - } - - void startObjectMember() - { - if (!isFirstInObj) { - putStr(","); - putIndent(); - } else { - isFirstInObj = false; - } - } - - void startArrayValue() - { - if (!isFirstInArray) { - putStr(","); - putIndent(); - } else { - isFirstInArray = false; - } - } - -public: - this(ref Chain chain) { - outputChain = &chain; - } - - this(ref Chain chain, int spacing, int indent) { - outputChain = &chain; - this.spacing = spacing; - this.indent = indent; - } - - void beginObject() { - isFirstInObj = true; - ++spacing; - putStr("{"); - putIndent(); - } - - void endObject() { - --spacing; - putIndent(); - putStr("}"); - } - - void beginArray() { - isFirstInArray = true; - ++spacing; - putStr("["); - putIndent(); - } - - void endArray() { - --spacing; - putIndent(); - putStr("]"); - } - - void addMember(T)(T key) { - addMember(key, memberOptions); - } - - void addMember(T)(T key, MemberOptions options) { - - startObjectMember(); - - static if (is(typeof(key) == string)) { - if (options.escapeKey) { - beginString(options.quoteChar); - addStringData(key); - endString(); - } else { - beginString(options.quoteChar); - addStringData!(false, false)(key); - endString(); - } - } else { - putStr(to!string(key)); - } - // Handle colon spacing based on enum value - final switch (options.colonSpacing) { - case MemberOptions.ColonSpacing.none: - putStr(":"); // "key":"value" - break; - case MemberOptions.ColonSpacing.after: - putStr(": "); // "key": "value" - break; - case MemberOptions.ColonSpacing.both: - putStr(" : "); // "key" : "value" - break; - } - - isFirstInObj = false; - } - - // expose startArrayValue to callers that emit arrays - void beginArrayValue() { - startArrayValue(); - } - - void beginString(char quoteChar = '"') { - if (quoteChar != '"' && quoteChar != '\'') - throw new JSONIopipeException("Invalid quote character"); - - currentQuoteChar = quoteChar; - putStr(("eChar)[0 .. 1]); - } - - // will automatically escape string data as needed. - void addStringData(bool validate = true, bool addEscapes = true, T)(T value) { - static if (validate) { - // Validate that the string doesn't contain invalid characters - foreach(char c; value) { - if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') { - throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); - } - } - } - - static if (addEscapes) { - import std.algorithm.iteration: substitute; - import std.conv: to; - auto escaped = value.substitute!(jsonEscapeSubstitutions!()).to!string; - putStr(escaped); - } else { - putStr(value); - } - } - - - void endString() { - putStr((¤tQuoteChar)[0 .. 1]); - } - - // allow numeric types, or strings that are validated to be JSON numbers - void addNumericData(T)(T value, string formatStr = "%s") { - static if (is(T == string)) { - if (!isValidJSONNumber(value)) { - throw new JSONIopipeException(format("Invalid JSON number: %s", value)); - } - putStr(value); - } else { - static assert(isNumeric!T, "addNumericData requires a numeric type"); - import std.format : formattedWrite; - formattedWrite(&putStr, formatStr, value); - } - } - - // null, true, false, inf, etc. - void addKeywordValue(KeywordValue value) { - putStr(value); - } - - void addWhitespace(T)(T data) { - foreach(char c; data) { - if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { - throw new JSONIopipeException(format("Invalid whitespace character: \\u%04X", cast(int)c)); - } - } - putStr(data); - } - - // // add a comment (JSON5 only), must be a complete comment (validated) - void addComment(T)(T commentData) { - static if (is(T == string)) { - import std.algorithm.searching : startsWith, endsWith; - if (!commentData.startsWith("//") && !commentData.startsWith("/*") && !commentData.endsWith("*/")) { - throw new JSONIopipeException(format("Invalid comment format: %s", commentData)); - } - } - putStr(commentData); - } - - void flushWritten() { - outputChain.release(nWritten); - nWritten = 0; - } - - auto chain() { - return outputChain; - } +auto peekSkipComma(JT)(ref JT tokenizer) +{ + auto token = tokenizer.peekSignificant(); + if(token != JSONToken.Comma) + return token; + // consume the comma + cast(void)tokenizer.nextSignificant; + return tokenizer.peekSignificant(); } From 956bd942df5a6aca0dcbe6bcae4ddbe05fe55628 Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Wed, 26 Nov 2025 08:46:28 -0700 Subject: [PATCH 7/9] delete duplicated peekComma() definition --- source/iopipe/json/serialize.d | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index 6f2acf3..a6fd999 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -31,18 +31,6 @@ import std.typecons : Nullable; import std.conv; import std.format; - -auto peekSkipComma(JT)(ref JT tokenizer) -{ - auto token = tokenizer.peekSignificant(); - if(token != JSONToken.Comma) - return token; - // consume the comma - cast(void)tokenizer.nextSignificant; - return tokenizer.peekSignificant(); -} - - struct DefaultDeserializationPolicy(bool caseInsensitive = false) { ReleasePolicy relPol = ReleasePolicy.afterMembers; // default policy int maxDepthAvailable = 64; From 4c670cca31636ebe0872e46734b03a6eb09a675a Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Tue, 2 Dec 2025 21:10:53 -0700 Subject: [PATCH 8/9] use canDoXXX before doing something --- examples/formatjson/formatjson.d | 1 - source/iopipe/json/formatter.d | 559 ++++++++++++++++++++----------- source/iopipe/json/serialize.d | 2 +- 3 files changed, 368 insertions(+), 194 deletions(-) diff --git a/examples/formatjson/formatjson.d b/examples/formatjson/formatjson.d index 12e1f9f..07762c2 100644 --- a/examples/formatjson/formatjson.d +++ b/examples/formatjson/formatjson.d @@ -102,7 +102,6 @@ void main(string[] args) auto formatter = JSONFormatter!(typeof(outputter))(outputter); serializeWithFormatter(formatter, jsonData); - formatter.addWhitespace("\n"); formatter.flushWritten(); } diff --git a/source/iopipe/json/formatter.d b/source/iopipe/json/formatter.d index af5afbb..22913a5 100644 --- a/source/iopipe/json/formatter.d +++ b/source/iopipe/json/formatter.d @@ -4,6 +4,7 @@ public import iopipe.json.common; import iopipe.traits : isIopipe, WindowType; import iopipe.bufpipe; +import iopipe.json.serialize : jsonEscapeSubstitutions; import std.bitmanip : BitArray; import std.exception : enforce; @@ -14,7 +15,6 @@ import std.algorithm.iteration : substitute; import std.algorithm.searching : startsWith, endsWith; import std.format; - struct MemberOptions { char quoteChar = '"'; // Use " or ' for quotes bool escapeKey = true; // Whether to escape special chars in key @@ -49,13 +49,29 @@ private: MemberOptions memberOptions; size_t nWritten = 0; - char currentQuoteChar = '"'; + char currentQuoteChar = char.init; private enum State : ubyte { Begin, // next item should be either an Object or Array or String - First, // Just started a new object or array. - Member, // Expect next member (name for object, value for array_ + First, // Expect the first member of a new object or array. + /* + * In an object: + * - First + * - Value(key) + * - Value(value) + * + * - Member + * - Value(key) + * - Value(value) + * ... + * + * In an array: + * - First(value) + * - Member(value) + * ... + */ + Member, // Expect next member (key for object, value for array) Value, // Expect value End // there shouldn't be any more items } @@ -65,11 +81,85 @@ private: size_t stackLen; State state = State.Begin; - bool inObj() @property + bool inObj() const @property nothrow { return stackLen == 0 ? false : stack[stackLen - 1]; } + // Can we add a value (string, number, keyword, begin object/array)? + bool canAddValue() const nothrow + { + final switch (state) with (State) + { + case Begin: + return true; // root value + case First: + // only allowed for arrays; for objects we must go through addMember + return !inObj(); + case Member: + // only allowed for arrays; for objects we must go through addMember + return !inObj(); + case Value: + return true; + case End: + return false; + } + } + + // Can we add an object member (key)? + // If so, add comma/indent as needed. + bool canAddObjMember() + { + if (!inObj()) + return false; + final switch (state) with (State) + { + case First: + // no need to change state to Member, any key string added later will do that + return true; + case Member: + putStr(","); + putIndent(); + return true; + case Begin: + case Value: + case End: + return false; + } + } + + // Can we add an array member? + // If so, add comma/indent as needed. + bool canAddArrayMember() + { + if (inObj()) + return false; + final switch (state) with (State) + { + case First: + // no need to change state to Member, any value added later will do that + return true; + case Member: + putStr(","); + putIndent(); + return true; + case Begin: + case Value: + case End: + return false; + } + } + + bool canCloseObject() const nothrow + { + return inObj() && (state == State.Member || state == State.First); + } + + bool canCloseArray() const nothrow + { + return stackLen > 0 && !inObj() && (state == State.Member || state == State.First); + } + void pushContainer(bool isObj) { if (stackLen == stack.length) @@ -82,16 +172,20 @@ private: void popContainer() { state = (--stackLen == 0) ? State.End : State.Member; + if (state == state.End) + putIndent(); } - void putIndent() { + void putIndent() + { outputChain.ensureElems(nWritten + indent * spacing + 1); outputChain.window[nWritten] = '\n'; outputChain.window[nWritten + 1 .. nWritten + 1 + indent * spacing] = ' '; nWritten += indent * spacing + 1; } - void putStr(const(char)[] s) { + void putStr(const(char)[] s) + { outputChain.ensureElems(nWritten + s.length); outputChain.window[nWritten .. nWritten + s.length] = s; nWritten += s.length; @@ -102,55 +196,6 @@ private: return !str.matchFirst(numberRegex).empty; } - void startObjectMember() - { - enforce(inObj(), "startObjectMember only valid inside object"); - - final switch (state) with (State) - { - case First: - // first member: no comma - state = Member; - break; - case Member: - // subsequent member: comma + indent - putStr(","); - putIndent(); - // stay in Member - break; - case Begin, Value, End: - throw new JSONIopipeException( - "startObjectMember not allowed in current state"); - } - } - - void startArrayValue() - { - enforce(!inObj(), "startArrayValue only valid inside array"); - - final switch (state) with (State) - { - case First: - // first element: no comma - state = Member; - break; - case Member: - // subsequent element: comma + indent - putStr(","); - putIndent(); - // stay in Member - break; - case Begin, Value, End: - throw new JSONIopipeException( - "startArrayValue not allowed in current state"); - } - } - - private void endStringRaw() { - putStr((¤tQuoteChar)[0 .. 1]); - // no state change - } - public: this(ref Chain chain) { outputChain = &chain; @@ -163,15 +208,9 @@ public: } void beginObject() { - final switch (state) with (State) - { - case Begin, Member, Value: - break; - case First, End: - throw new JSONIopipeException( - "beginObject not allowed in current state"); - } - + if (!canAddValue()) + throw new JSONIopipeException("beginObject not allowed in current state"); + pushContainer(true); // object state = State.First; // just started @@ -181,24 +220,19 @@ public: } void endObject() { - enforce(inObj(), "endObject outside object"); - + if (!canCloseObject()) + throw new JSONIopipeException("endObject not allowed in current state"); + --spacing; putIndent(); putStr("}"); - popContainer(); // State.End or State.Comma + popContainer(); // State.End or State.Member } void beginArray() { - final switch (state) with (State) - { - case Begin, Member, Value: - break; - case First, End: - throw new JSONIopipeException( - "beginArray not allowed in current state"); - } + if (!canAddValue()) + throw new JSONIopipeException("beginArray not allowed in current state"); pushContainer(false); // array state = State.First; @@ -209,7 +243,8 @@ public: } void endArray() { - enforce(!inObj(), "endArray outside array"); + if (!canCloseArray()) + throw new JSONIopipeException("endArray not allowed in current state"); --spacing; putIndent(); @@ -218,30 +253,25 @@ public: popContainer(); } - // Public hook for arrays: call before each element + // call before each element void beginArrayValue() { - startArrayValue(); + if (!canAddArrayMember()) + throw new JSONIopipeException("beginArrayValue not allowed in current state"); } void addMember(T)(T key) { addMember(key, memberOptions); } + // First/Member -> Value(before string key) -> Member(after string key) -> Value(before value) -> Member(after value) ... void addMember(T)(T key, MemberOptions options) { - enforce(inObj(), "addMember only valid inside object"); - - final switch (state) with (State) - { - case First, Member: - break; - case Begin, Value, End: - throw new JSONIopipeException( - "addMember not allowed in current state"); - } + if (!canAddObjMember()) + throw new JSONIopipeException("addMember not allowed in current state"); - startObjectMember(); + // prepare for a key string + state = State.Value; static if (is(typeof(key) == string)) { @@ -249,13 +279,13 @@ public: { beginString(options.quoteChar); addStringData(key); - endStringRaw(); + endString(); } else { beginString(options.quoteChar); addStringData!(false, false)(key); - endStringRaw(); + endString(); } } else @@ -276,19 +306,16 @@ public: break; } + // prepare for a key's value state = State.Value; } void beginString(char quoteChar = '"') { - // value allowed at: Begin (root), Member (array element), Value (object value) - final switch (state) with (State) - { - case Begin, Member, Value: - break; - case First, End: - throw new JSONIopipeException( - "beginString not allowed in current state"); - } + if (!canAddValue()) + throw new JSONIopipeException("beginString not allowed in current state"); + + if (currentQuoteChar != char.init) + throw new JSONIopipeException("nested beginString not allowed"); if (quoteChar != '"' && quoteChar != '\'') throw new JSONIopipeException("Invalid quote character"); @@ -299,6 +326,9 @@ public: // will automatically escape string data as needed. void addStringData(bool validate = true, bool addEscapes = true, T)(T value) { + if (currentQuoteChar == char.init) + throw new JSONIopipeException("cannot add string data outside of beginString/endString"); + static if (validate) { // Validate that the string doesn't contain invalid characters foreach(char c; value) { @@ -310,19 +340,40 @@ public: static if (addEscapes) { auto escaped = value.substitute!(jsonEscapeSubstitutions!()).to!string; - putStr(escaped); + + // if we started with a single quote, also escape any remaining ' + if (currentQuoteChar == '\'') { + import std.array : appender; + auto app = appender!string(); + foreach (char c; escaped) { + if (c == '\'') { + app.put('\\'); + app.put('\''); + } else { + app.put(c); + } + } + putStr(app.data); + } else { + // normal JSON style (double quotes only) + putStr(escaped); + } } else { putStr(value); } } - void endString() { + if (currentQuoteChar == char.init) + throw new JSONIopipeException("cannot endString without a matching beginString"); + putStr((¤tQuoteChar)[0 .. 1]); + currentQuoteChar = char.init; - if (stackLen == 0) + if (stackLen == 0) { state = State.End; - else + putIndent(); + } else // If it's at the end of an array or an object, // endArray() and endObject() will set the state appropriately state = State.Member; @@ -330,15 +381,8 @@ public: // allow numeric types, or strings that are validated to be JSON numbers void addNumericData(T)(T value, string formatStr = "%s") { - - final switch (state) with (State) - { - case Begin, Member, Value: - break; - case First, End: - throw new JSONIopipeException( - "addNumericData not allowed in current state"); - } + if (!canAddValue()) + throw new JSONIopipeException("addNumericData not allowed in current state"); static if (is(T == string)) { if (!isValidJSONNumber(value)) { @@ -358,14 +402,8 @@ public: // null, true, false, inf, etc. void addKeywordValue(KeywordValue value) { - final switch (state) with (State) - { - case Begin, Member, Value: - break; - case First, End: - throw new JSONIopipeException( - "addKeywordValue not allowed in current state"); - } + if (!canAddValue()) + throw new JSONIopipeException("addKeywordValue not allowed in current state"); putStr(value); @@ -375,25 +413,6 @@ public: state = State.Member; } - void addWhitespace(T)(T data) { - foreach(char c; data) { - if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { - throw new JSONIopipeException(format("Invalid whitespace character: \\u%04X", cast(int)c)); - } - } - putStr(data); - } - - // // add a comment (JSON5 only), must be a complete comment (validated) - void addComment(T)(T commentData) { - static if (is(T == string)) { - if (!commentData.startsWith("//") && !commentData.startsWith("/*") && !commentData.endsWith("*/")) { - throw new JSONIopipeException(format("Invalid comment format: %s", commentData)); - } - } - putStr(commentData); - } - void flushWritten() { outputChain.release(nWritten); nWritten = 0; @@ -405,62 +424,218 @@ public: } -/** Eponymous template that generates an argument list for std.algorithm.substitute to correctly escape json strings - * Result: - * AliasSeq!("", "", "", "", ...); - * - * copied from serialize.d - */ -private template jsonEscapeSubstitutions() +private struct TestChain { - import std.algorithm.iteration; - import std.range; - import std.ascii: ControlChar; - - // Control characters [0,31 aka 0x1f] + 2 special characters '"' and '\' - private enum charactersToEscape = chain(only('\"', '\\'), iota(0x1f + 1)); - - private struct JsonEscapeMapping { - char chr; - string escapeSequence; - } - - /* Special characters we have to (in case of '"' and '\') per the spec or want to escape seperately for readability - * Everything else gets converted to the "\uxxxx" unicode escape - */ - private enum JsonEscapeMapping[7] JSON_ESCAPES = [ - {'\"', `\"`}, - {'\\', `\\`}, - {ControlChar.bs, `\b`}, - {ControlChar.lf, `\n`}, - {ControlChar.cr, `\r`}, - {ControlChar.tab, `\t`}, - {ControlChar.ff, `\f`}, - ]; - - private JsonEscapeMapping escapeSingleChar(int c) + char[] buf; + + @property char[] window() { - switch(c) - { - static foreach(e; JSON_ESCAPES) - { - case e.chr: - return e; - } - default: - return JsonEscapeMapping(cast(char)c, format!`\u%04x`(c)); - } + return buf; + } + + // Grow buffer when formatter calls ensureElems via free function + size_t extend(size_t elements) + { + import core.memory : GC; + + // extend is called by ensureElems until window.length >= requested + auto oldLen = buf.length; + auto newLen = oldLen + elements; + auto p = cast(char*)GC.malloc(newLen * char.sizeof); + auto newBuf = p[0 .. newLen]; + + if (oldLen) + newBuf[0 .. oldLen] = buf[]; + + buf = newBuf; + return elements; } - // Convert struct to AliasSeq with correct types so it can be used as parameters of a function - private template UnpackStruct(JsonEscapeMapping jem) + void release(size_t /*elements*/) { - // Must convert char to string for use with substitute - enum string staticCast(char c) = c.to!string; - enum string staticCast(string s) = s; - alias UnpackStruct = staticMap!(staticCast, jem.tupleof); + // formatter calls release(nWritten); for tests we can ignore it } +} + +unittest +{ + import iopipe.traits : isIopipe; + import iopipe.bufpipe : ensureElems; + import std.string : strip; + import std.stdio : writeln; + + TestChain chain; + static assert(isIopipe!TestChain); + + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginObject(); + fmt.addMember("a"); + fmt.addNumericData(1); + fmt.endObject(); + fmt.flushWritten(); + + auto s = cast(string)chain.buf.strip; + assert(s == "{\n \"a\": 1\n}"); +} + +unittest +{ + // Simple array: [1,2,3] + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginArray(); + fmt.beginArrayValue(); + fmt.addNumericData(1); + fmt.beginArrayValue(); + fmt.addNumericData(2); + fmt.beginArrayValue(); + fmt.addNumericData(3); + fmt.endArray(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + assert(s == "[\n 1,\n 2,\n 3\n]"); +} + +unittest +{ + // String with escapes and single‑quote behavior + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + // double‑quoted: inner " and \ should be escaped + fmt.beginString('"'); + fmt.addStringData(`He said: "hi" \ test`); + fmt.endString(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + assert(s == `"He said: \"hi\" \\ test"`); + + // reset chain, now test single‑quoted: inner ' must be escaped as \' + chain.buf.length = 0; + auto fmt2 = JSONFormatter!(TestChain)(chain); + + fmt2.beginString('\''); + fmt2.addStringData(`It's ok`); + fmt2.endString(); + fmt2.flushWritten(); + + s = cast(string)chain.buf.strip; + assert(s == `'It\'s ok'`); +} + +unittest +{ + // Keywords: null/true/false + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginArray(); + fmt.beginArrayValue(); + fmt.addKeywordValue(KeywordValue.Null); + fmt.beginArrayValue(); + fmt.addKeywordValue(KeywordValue.True); + fmt.beginArrayValue(); + fmt.addKeywordValue(KeywordValue.False); + fmt.endArray(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + assert(s == "[\n null,\n true,\n false\n]"); +} + +unittest +{ + // ERROR: endArray without beginArray + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.endArray()); +} + +unittest +{ + // ERROR: addMember when not inside object + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.addMember("a")); +} + +unittest +{ + // ERROR: add key string directily without in addMember + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + fmt.beginObject(); + + assertThrown!JSONIopipeException(fmt.beginString()); +} + +unittest +{ + // ERROR: addArrayValue when not inside array + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.beginArrayValue()); +} + +unittest +{ + // ERROR: nested beginString + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginString(); + assertThrown!JSONIopipeException(fmt.beginString()); +} + +unittest +{ + // ERROR: addStringData without beginString + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.addStringData("oops")); +} + +unittest +{ + // ERROR: endString without beginString + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + assertThrown!JSONIopipeException(fmt.endString()); +} + +unittest +{ + // ERROR: invalid JSON number string + import std.exception : assertThrown; + + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); - import std.meta; - private alias jsonEscapeSubstitutions = staticMap!(UnpackStruct, aliasSeqOf!(charactersToEscape.map!escapeSingleChar)); + assertThrown!JSONIopipeException(fmt.addNumericData("01")); } \ No newline at end of file diff --git a/source/iopipe/json/serialize.d b/source/iopipe/json/serialize.d index a6fd999..c352c60 100644 --- a/source/iopipe/json/serialize.d +++ b/source/iopipe/json/serialize.d @@ -1888,7 +1888,7 @@ unittest * Result: * AliasSeq!("", "", "", "", ...); */ -private template jsonEscapeSubstitutions() +package template jsonEscapeSubstitutions() { import std.algorithm.iteration; import std.range; From 090a5e0bf86b7d06da361f9f3142722d0a29cf95 Mon Sep 17 00:00:00 2001 From: gulugulubing <413153391@qq.com> Date: Thu, 25 Dec 2025 20:38:02 -0700 Subject: [PATCH 9/9] Correct addStringData --- source/iopipe/json/formatter.d | 257 ++++++++++++++++++++++++++++----- 1 file changed, 224 insertions(+), 33 deletions(-) diff --git a/source/iopipe/json/formatter.d b/source/iopipe/json/formatter.d index 22913a5..1ee43f1 100644 --- a/source/iopipe/json/formatter.d +++ b/source/iopipe/json/formatter.d @@ -51,9 +51,9 @@ private: size_t nWritten = 0; char currentQuoteChar = char.init; - private enum State : ubyte + enum State : ubyte { - Begin, // next item should be either an Object or Array or String + Begin, // next item should be either an Object or Array or String or Number or Keyword First, // Expect the first member of a new object or array. /* * In an object: @@ -76,6 +76,14 @@ private: End // there shouldn't be any more items } + // How to treat string data passed to addStringData + enum StringMode : ubyte + { + passThru, // do not modify or validate the contents + addEscapes, // escape any characters that would be invalid in JSON (default) + validate // validate that existing escapes / characters are JSON-valid + } + // 0 = array, 1 = object (same convention as parser) BitArray stack; size_t stackLen; @@ -89,6 +97,9 @@ private: // Can we add a value (string, number, keyword, begin object/array)? bool canAddValue() const nothrow { + if (currentQuoteChar != char.init) + return false; // can't add value inside a string + final switch (state) with (State) { case Begin: @@ -115,7 +126,6 @@ private: final switch (state) with (State) { case First: - // no need to change state to Member, any key string added later will do that return true; case Member: putStr(","); @@ -137,7 +147,6 @@ private: final switch (state) with (State) { case First: - // no need to change state to Member, any value added later will do that return true; case Member: putStr(","); @@ -273,25 +282,19 @@ public: // prepare for a key string state = State.Value; - static if (is(typeof(key) == string)) + beginString(options.quoteChar); + static if (__traits(compiles, putStr(key))) { if (options.escapeKey) - { - beginString(options.quoteChar); addStringData(key); - endString(); - } else - { - beginString(options.quoteChar); - addStringData!(false, false)(key); - endString(); - } + addStringData!(StringMode.passThru)(key); } else { putStr(to!string(key)); } + endString(); final switch (options.colonSpacing) { @@ -324,41 +327,108 @@ public: putStr(("eChar)[0 .. 1]); } - // will automatically escape string data as needed. - void addStringData(bool validate = true, bool addEscapes = true, T)(T value) { + void addStringData(StringMode mode = StringMode.addEscapes, T)(T value) { if (currentQuoteChar == char.init) throw new JSONIopipeException("cannot add string data outside of beginString/endString"); - static if (validate) { - // Validate that the string doesn't contain invalid characters - foreach(char c; value) { - if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') { - throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + static if (mode == StringMode.validate) + { + import std.ascii : isHexDigit; + + auto s = value.to!string; // ensure we can index / look ahead + + size_t i = 0; + while (i < s.length) + { + immutable c = s[i]; + + if (c == '\\') + { + if (i + 1 >= s.length) + throw new JSONIopipeException("Invalid escape sequence: lone backslash at end of string"); + + immutable next = s[i + 1]; + + // Allow \', but only when we're using single-quoted strings + if (next == '\'' && currentQuoteChar == '\'') + { + i += 2; + continue; + } + + // Standard JSON escapes + switch (next) + { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + i += 2; + continue; + case 'u': + if (i + 5 >= s.length) + throw new JSONIopipeException("Invalid \\u escape: too short"); + + foreach (j; 2 .. 6) + { + immutable h = s[i + j]; + if (!isHexDigit(h)) + throw new JSONIopipeException("Invalid \\u escape: non-hex digit"); + } + + i += 6; // '\\', 'u', and 4 hex digits + continue; + default: + throw new JSONIopipeException("Invalid escape sequence in string"); + } + } + else + { + // Active quote character must be escaped + if (c == currentQuoteChar) + throw new JSONIopipeException("Unescaped quote character in string"); + + // Bare control characters must not appear; they must be escaped + if (c < 0x20) + throw new JSONIopipeException(format("Invalid control character \\u%04X in string", cast(int)c)); + + ++i; } } + + // In validate mode we output the string unchanged + putStr(s); } - - static if (addEscapes) { + else static if (mode == StringMode.addEscapes) + { + // Escape any characters that would be invalid in JSON auto escaped = value.substitute!(jsonEscapeSubstitutions!()).to!string; - + // if we started with a single quote, also escape any remaining ' - if (currentQuoteChar == '\'') { + if (currentQuoteChar == '\'') + { import std.array : appender; auto app = appender!string(); - foreach (char c; escaped) { - if (c == '\'') { + foreach (char c; escaped) + { + if (c == '\'') + { app.put('\\'); app.put('\''); - } else { + } + else + { app.put(c); } } putStr(app.data); - } else { + } + else + { // normal JSON style (double quotes only) putStr(escaped); } - } else { + } + else static if (mode == StringMode.passThru) + { + // Assume the caller already provided a valid JSON string fragment putStr(value); } } @@ -394,9 +464,10 @@ public: formattedWrite(&putStr, formatStr, value); } - if (stackLen == 0) + if (stackLen == 0) { state = State.End; - else + putIndent(); + } else state = State.Member; } @@ -407,8 +478,10 @@ public: putStr(value); - if (stackLen == 0) + if (stackLen == 0) { state = State.End; + putIndent(); + } else state = State.Member; } @@ -529,6 +602,124 @@ unittest assert(s == `'It\'s ok'`); } +unittest +{ + // addEscapes: real newline becomes literal "\n" in JSON + TestChain chain; + auto fmt = JSONFormatter!(TestChain)(chain); + + fmt.beginString('"'); + fmt.addStringData("line1\nline2"); // contains an actual newline character + fmt.endString(); + fmt.flushWritten(); + + import std.string : strip; + auto s = cast(string)chain.buf.strip; + // Expect JSON text with a backslash-n escape, not a raw newline + assert(s == `"line1\nline2"`); +} + +unittest +{ + // StringMode.validate: accept only correctly escaped content + import std.exception : assertThrown; + import std.string : strip; + + alias Fmt = JSONFormatter!(TestChain); + alias SM = Fmt.StringMode; + + TestChain chain; + + // 1) Double‑quoted string with valid escapes should pass unchanged + auto fmt = Fmt(chain); + fmt.beginString('"'); + fmt.addStringData!(SM.validate)(`He said: \"hi\" \\ test`); + fmt.endString(); + fmt.flushWritten(); + + auto s = cast(string)chain.buf.strip; + assert(s == `"He said: \"hi\" \\ test"`); + + // 2) Invalid escape ("\q") should throw in validate mode + chain.buf.length = 0; + auto fmt2 = Fmt(chain); + fmt2.beginString('"'); + assertThrown!JSONIopipeException(fmt2.addStringData!(SM.validate)(`He said: \q`)); + + // 3) Lone backslash at end should throw in validate mode + chain.buf.length = 0; + auto fmt3 = Fmt(chain); + fmt3.beginString('"'); + assertThrown!JSONIopipeException(fmt3.addStringData!(SM.validate)(`oops \`)); + + // 4) Unescaped quote should throw in validate mode + chain.buf.length = 0; + auto fmt4 = Fmt(chain); + fmt4.beginString('"'); + assertThrown!JSONIopipeException(fmt4.addStringData!(SM.validate)(`He said: "hi"`)); + + // 5) Single‑quoted validate: \' is allowed, bare ' is not + chain.buf.length = 0; + auto fmt5 = Fmt(chain); + fmt5.beginString('\''); + fmt5.addStringData!(SM.validate)(`It\'s ok`); + fmt5.endString(); + fmt5.flushWritten(); + + s = cast(string)chain.buf.strip; + assert(s == `'It\'s ok'`); + + chain.buf.length = 0; + auto fmt6 = Fmt(chain); + fmt6.beginString('\''); + assertThrown!JSONIopipeException(fmt6.addStringData!(SM.validate)(`It's bad`)); +} + +unittest +{ + // StringMode.validate: cover all standard JSON escapes and \u forms + import std.exception : assertThrown; + + alias Fmt = JSONFormatter!(TestChain); + alias SM = Fmt.StringMode; + + TestChain chain; + + // 1) All standard single-character escapes should be accepted + auto fmt = Fmt(chain); + fmt.beginString('"'); + fmt.addStringData!(SM.validate)(`\" \\ \/ \b \f \n \r \t`); + fmt.endString(); + fmt.flushWritten(); + + // 2) Valid \u escape should be accepted + chain.buf.length = 0; + auto fmt2 = Fmt(chain); + fmt2.beginString('"'); + fmt2.addStringData!(SM.validate)(`\u00AF`); + fmt2.endString(); + fmt2.flushWritten(); + + // 3) Too-short \u escape should throw + chain.buf.length = 0; + auto fmt3 = Fmt(chain); + fmt3.beginString('"'); + assertThrown!JSONIopipeException(fmt3.addStringData!(SM.validate)(`\u12`)); + + // 4) \u escape with non-hex digits should throw + chain.buf.length = 0; + auto fmt4 = Fmt(chain); + fmt4.beginString('"'); + assertThrown!JSONIopipeException(fmt4.addStringData!(SM.validate)(`\u12xz`)); + + // 5) Bare control character (< 0x20) should throw + chain.buf.length = 0; + auto fmt5 = Fmt(chain); + fmt5.beginString('"'); + string bad = "ok" ~ "\x01"; // embed a real control char 0x01 + assertThrown!JSONIopipeException(fmt5.addStringData!(SM.validate)(bad)); +} + unittest { // Keywords: null/true/false