fix: edge case when separator is whitespace and there are whitespaces…

… within the objects
juanjoDiaz · Apr 22, 2024 · 818578f · 818578f
1 parent ef9a919
commit 818578f
Show file tree

Hide file tree

Showing 17 changed files with 170 additions and 24 deletions.
diff --git a/packages/node/dist/deno/README.md b/packages/node/dist/deno/README.md
@@ -182,7 +182,7 @@ Imagine an endpoint that send a large amount of JSON objects one after the other
 
 ## License
 
-See [LICENSE.md].
+See [LICENSE.md](../../LICENSE).
 
 [npm-version-badge]: https://badge.fury.io/js/@streamparser%2Fjson-node.svg
 [npm-badge-url]: https://www.npmjs.com/package/@streamparser/json-node

diff --git a/packages/node/dist/deno/index.ts b/packages/node/dist/deno/index.ts
@@ -10,4 +10,4 @@ export {
   TokenParserMode,
   type StackElement,
   TokenType,
-} from "https://deno.land/x/[email protected].19/index.ts";
+} from "https://deno.land/x/[email protected].20/index.ts";
diff --git a/packages/node/dist/deno/jsonparser.ts b/packages/node/dist/deno/jsonparser.ts
@@ -3,7 +3,7 @@ import {
   type TransformOptions,
   type TransformCallback,
 } from "stream";
-import { JSONParser, type JSONParserOptions } from "https://deno.land/x/[email protected].19/index.ts";
+import { JSONParser, type JSONParserOptions } from "https://deno.land/x/[email protected].20/index.ts";
 
 export default class JSONParserTransform extends Transform {
   private jsonParser: JSONParser;

diff --git a/packages/node/dist/deno/tokenizer.ts b/packages/node/dist/deno/tokenizer.ts
@@ -5,7 +5,7 @@ import {
 } from "stream";
 import Tokenizer, {
   type TokenizerOptions,
-} from "https://deno.land/x/[email protected].19/tokenizer.ts";
+} from "https://deno.land/x/[email protected].20/tokenizer.ts";
 
 export default class TokenizerTransform extends Transform {
   private tokenizer: Tokenizer;

diff --git a/packages/node/dist/deno/tokenparser.ts b/packages/node/dist/deno/tokenparser.ts
@@ -3,7 +3,7 @@ import {
   type TransformOptions,
   type TransformCallback,
 } from "stream";
-import { TokenParser, type TokenParserOptions } from "https://deno.land/x/[email protected].19/index.ts";
+import { TokenParser, type TokenParserOptions } from "https://deno.land/x/[email protected].20/index.ts";
 
 export default class TokenParserTransform extends Transform {
   private tokenParser: TokenParser;

diff --git a/packages/node/test/separator.ts b/packages/node/test/separator.ts
@@ -40,6 +40,22 @@ describe("separator", () => {
     });
   });
 
+  test("support multiple whitespace separators", async () => {
+    let i = 0;
+    const value = "1 2\t3\n4\n\r5 \n6\n\n7\n\r\n\r8 \t\n\n\r9";
+    const expected = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+    const separator = "";
+
+    await runJSONParserTest(
+      new JSONParser({ separator }),
+      value,
+      ({ value }) => {
+        expect(value).toEqual(expected[i]);
+        i += 1;
+      },
+    );
+  });
+
   test(`separator: fail on invalid value`, async () => {
     try {
       await runJSONParserTest(new JSONParser({ separator: "abc" }), ["abe"]);
@@ -75,4 +91,24 @@ describe("separator", () => {
       );
     }
   });
+
+  test("not fail when whitespaces match separator", async () => {
+    let i = 0;
+    const value = `{
+      "a": 0,
+      "b": 1,
+      "c": -1
+    }`;
+    const expected = [0, 1, -1, { a: 0, b: 1, c: -1 }];
+    const separator = "\n";
+
+    await runJSONParserTest(
+      new JSONParser({ separator }),
+      value,
+      ({ value }) => {
+        expect(value).toEqual(expected[i]);
+        i += 1;
+      },
+    );
+  });
 });
diff --git a/packages/plainjs/dist/deno/README.md b/packages/plainjs/dist/deno/README.md
@@ -10,7 +10,7 @@ Fast dependency-free library to parse a JSON stream using utf-8 encoding in Node
 *tldr;*
 
 ```javascript
-import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
 const parser = new JSONParser();
 parser.onValue = ({ value }) => { /* process data */ };
@@ -52,7 +52,7 @@ If you are targeting browsers or systems in which these might be missing, you ne
 A JSON compliant tokenizer that parses a utf-8 stream into JSON tokens
 
 ```javascript
-import { Tokenizer } from "https://deno.land/x/[email protected].19/index.ts";/
+import { Tokenizer } from "https://deno.land/x/[email protected].20/index.ts";/
 
 const tokenizer = new Tokenizer(opts);
 ```
@@ -165,7 +165,7 @@ A drop-in replacement of `JSONparse` (with few ~~breaking changes~~ improvements
 
 
 ```javascript
-import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
 const parser = new JSONParser();
 ```
@@ -225,7 +225,7 @@ You push data using the `write` method which takes a string or an array-like obj
 You can subscribe to the resulting data using the 
 
 ```javascript
-import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
 const parser = new JSONParser({ stringBufferSize: undefined, paths: ['$'] });
 parser.onValue = console.log;
@@ -243,7 +243,7 @@ parser.write('"');// logs "Hello world!"
 Write is always a synchronous operation so any error during the parsing of the stream will be thrown during the write operation. After an error, the parser can't continue parsing.
 
 ```javascript
-import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
 const parser = new JSONParser({ stringBufferSize: undefined });
 parser.onValue = console.log;
@@ -258,7 +258,7 @@ try {
 You can also handle errors using callbacks:
 
 ```javascript
-import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
 const parser = new JSONParser({ stringBufferSize: undefined });
 parser.onValue = console.log;
@@ -296,7 +296,7 @@ Imagine an endpoint that send a large amount of JSON objects one after the other
 Imagine an endpoint that send a large amount of JSON objects one after the other (`[{"id":1},{"id":2},{"id":3},...]`).
 
 ```js
-  import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+  import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
   const jsonparser = new JSONParser({ stringBufferSize: undefined, paths: ['$.*'] });
   jsonparser.onValue = ({ value, key, parent, stack }) => {
@@ -317,7 +317,7 @@ Imagine an endpoint that send a large amount of JSON objects one after the other
 Imagine an endpoint that send a large amount of JSON objects one after the other (`"Once upon a midnight <...>"`).
 
 ```js
-  import { JSONParser } from "https://deno.land/x/[email protected].19/index.ts";/
+  import { JSONParser } from "https://deno.land/x/[email protected].20/index.ts";/
 
   const jsonparser = new JSONParser({ emitPartialTokens: true, emitPartialValues: true });
   jsonparser.onValue = ({ value, key, parent, stack, partial }) => {
@@ -366,7 +366,7 @@ jsonparser.onValue = (value, key, parent, stack) => {
 
 ## License
 
-See [LICENSE.md].
+See [LICENSE.md](../../LICENSE).
 
 [npm-version-badge]: https://badge.fury.io/js/@streamparser%2Fjson.svg
 [npm-badge-url]: https://www.npmjs.com/package/@streamparser/json

diff --git a/packages/plainjs/dist/deno/tokenparser.ts b/packages/plainjs/dist/deno/tokenparser.ts
@@ -1,3 +1,4 @@
+import { charset } from "./utils/utf-8.ts";
 import TokenType from "./utils/types/tokenType.ts";
 import type {
   JsonPrimitive,
@@ -325,6 +326,24 @@ export default class TokenParser {
         }
       }
 
+      // Edge case in which the separator is just whitespace and it's found in the middle of the JSON
+      if (
+        token === TokenType.SEPARATOR &&
+        this.state !== TokenParserState.SEPARATOR &&
+        Array.from(value as string)
+          .map((n) => n.charCodeAt(0))
+          .every(
+            (n) =>
+              n === charset.SPACE ||
+              n === charset.NEWLINE ||
+              n === charset.CARRIAGE_RETURN ||
+              n === charset.TAB,
+          )
+      ) {
+        // whitespace
+        return;
+      }
+
       throw new TokenParserError(
         `Unexpected ${TokenType[token]} (${JSON.stringify(
           value,

diff --git a/packages/plainjs/src/tokenparser.ts b/packages/plainjs/src/tokenparser.ts
@@ -1,3 +1,4 @@
+import { charset } from "./utils/utf-8.js";
 import TokenType from "./utils/types/tokenType.js";
 import type {
   JsonPrimitive,
@@ -325,6 +326,24 @@ export default class TokenParser {
         }
       }
 
+      // Edge case in which the separator is just whitespace and it's found in the middle of the JSON
+      if (
+        token === TokenType.SEPARATOR &&
+        this.state !== TokenParserState.SEPARATOR &&
+        Array.from(value as string)
+          .map((n) => n.charCodeAt(0))
+          .every(
+            (n) =>
+              n === charset.SPACE ||
+              n === charset.NEWLINE ||
+              n === charset.CARRIAGE_RETURN ||
+              n === charset.TAB,
+          )
+      ) {
+        // whitespace
+        return;
+      }
+
       throw new TokenParserError(
         `Unexpected ${TokenType[token]} (${JSON.stringify(
           value,

diff --git a/packages/plainjs/test/separator.ts b/packages/plainjs/test/separator.ts
@@ -40,6 +40,22 @@ describe("separator", () => {
     });
   });
 
+  test("support multiple whitespace separators", async () => {
+    let i = 0;
+    const value = "1 2\t3\n4\n\r5 \n6\n\n7\n\r\n\r8 \t\n\n\r9";
+    const expected = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+    const separator = "";
+
+    await runJSONParserTest(
+      new JSONParser({ separator }),
+      value,
+      ({ value }) => {
+        expect(value).toEqual(expected[i]);
+        i += 1;
+      },
+    );
+  });
+
   test(`separator: fail on invalid value`, async () => {
     try {
       await runJSONParserTest(new JSONParser({ separator: "abc" }), ["abe"]);
@@ -75,4 +91,24 @@ describe("separator", () => {
       );
     }
   });
+
+  test("not fail when whitespaces match separator", async () => {
+    let i = 0;
+    const value = `{
+      "a": 0,
+      "b": 1,
+      "c": -1
+    }`;
+    const expected = [0, 1, -1, { a: 0, b: 1, c: -1 }];
+    const separator = "\n";
+
+    await runJSONParserTest(
+      new JSONParser({ separator }),
+      value,
+      ({ value }) => {
+        expect(value).toEqual(expected[i]);
+        i += 1;
+      },
+    );
+  });
 });
diff --git a/packages/whatwg/dist/deno/README.md b/packages/whatwg/dist/deno/README.md
@@ -254,7 +254,7 @@ Imagine an endpoint that send a large amount of JSON objects one after the other
 
 ## License
 
-See [LICENSE.md].
+See [LICENSE.md](../../LICENSE).
 
 [npm-version-badge]: https://badge.fury.io/js/@streamparser%2Fjson-whatwg.svg
 [npm-badge-url]: https://www.npmjs.com/package/@streamparser/json-whatwg

diff --git a/packages/whatwg/dist/deno/index.ts b/packages/whatwg/dist/deno/index.ts
@@ -10,4 +10,4 @@ export {
   TokenParserMode,
   type StackElement,
   TokenType,
-} from "https://deno.land/x/[email protected].19/index.ts";
+} from "https://deno.land/x/[email protected].20/index.ts";
diff --git a/packages/whatwg/dist/deno/jsonparser.ts b/packages/whatwg/dist/deno/jsonparser.ts
@@ -1,5 +1,5 @@
-import { JSONParser, type JSONParserOptions } from "https://deno.land/x/[email protected].19/index.ts";
-import type { ParsedElementInfo } from "https://deno.land/x/[email protected].19/utils/types/parsedElementInfo.ts";
+import { JSONParser, type JSONParserOptions } from "https://deno.land/x/[email protected].20/index.ts";
+import type { ParsedElementInfo } from "https://deno.land/x/[email protected].20/utils/types/parsedElementInfo.ts";
 import { cloneParsedElementInfo } from "./utils.ts";
 
 class JSONParserTransformer

diff --git a/packages/whatwg/dist/deno/tokenizer.ts b/packages/whatwg/dist/deno/tokenizer.ts
@@ -1,7 +1,7 @@
 import Tokenizer, {
   type TokenizerOptions,
-} from "https://deno.land/x/[email protected].19/tokenizer.ts";
-import type { ParsedTokenInfo } from "https://deno.land/x/[email protected].19/utils/types/parsedTokenInfo.ts";
+} from "https://deno.land/x/[email protected].20/tokenizer.ts";
+import type { ParsedTokenInfo } from "https://deno.land/x/[email protected].20/utils/types/parsedTokenInfo.ts";
 
 class TokenizerTransformer
   extends Tokenizer

diff --git a/packages/whatwg/dist/deno/tokenparser.ts b/packages/whatwg/dist/deno/tokenparser.ts
@@ -1,6 +1,6 @@
-import { TokenParser, type TokenParserOptions } from "https://deno.land/x/[email protected].19/index.ts";
-import type { ParsedTokenInfo } from "https://deno.land/x/[email protected].19/utils/types/parsedTokenInfo.ts";
-import type { ParsedElementInfo } from "https://deno.land/x/[email protected].19/utils/types/parsedElementInfo.ts";
+import { TokenParser, type TokenParserOptions } from "https://deno.land/x/[email protected].20/index.ts";
+import type { ParsedTokenInfo } from "https://deno.land/x/[email protected].20/utils/types/parsedTokenInfo.ts";
+import type { ParsedElementInfo } from "https://deno.land/x/[email protected].20/utils/types/parsedElementInfo.ts";
 import { cloneParsedElementInfo } from "./utils.ts";
 
 class TokenParserTransformer

diff --git a/packages/whatwg/dist/deno/utils.ts b/packages/whatwg/dist/deno/utils.ts
@@ -1,4 +1,4 @@
-import type { ParsedElementInfo } from "https://deno.land/x/[email protected].19/utils/types/parsedElementInfo.ts";
+import type { ParsedElementInfo } from "https://deno.land/x/[email protected].20/utils/types/parsedElementInfo.ts";
 
 export function cloneParsedElementInfo(
   parsedElementInfo: ParsedElementInfo,

diff --git a/packages/whatwg/test/separator.ts b/packages/whatwg/test/separator.ts
@@ -40,6 +40,22 @@ describe("separator", () => {
     });
   });
 
+  test("support multiple whitespace separators", async () => {
+    let i = 0;
+    const value = "1 2\t3\n4\n\r5 \n6\n\n7\n\r\n\r8 \t\n\n\r9";
+    const expected = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+    const separator = "";
+
+    await runJSONParserTest(
+      new JSONParser({ separator }),
+      value,
+      ({ value }) => {
+        expect(value).toEqual(expected[i]);
+        i += 1;
+      },
+    );
+  });
+
   test(`separator: fail on invalid value`, async () => {
     try {
       await runJSONParserTest(new JSONParser({ separator: "abc" }), ["abe"]);
@@ -75,4 +91,24 @@ describe("separator", () => {
       );
     }
   });
+
+  test("not fail when whitespaces match separator", async () => {
+    let i = 0;
+    const value = `{
+      "a": 0,
+      "b": 1,
+      "c": -1
+    }`;
+    const expected = [0, 1, -1, { a: 0, b: 1, c: -1 }];
+    const separator = "\n";
+
+    await runJSONParserTest(
+      new JSONParser({ separator }),
+      value,
+      ({ value }) => {
+        expect(value).toEqual(expected[i]);
+        i += 1;
+      },
+    );
+  });
 });