```
├── .commitlintrc.json
├── .github/
├── workflows/
├── build.yaml (100 tokens)
├── .gitignore
├── .husky/
├── commit-msg
├── pre-commit
├── CHANGELOG.md (2.8k tokens)
├── LICENSE.md (100 tokens)
├── README.md (1600 tokens)
├── babel-cjs.config.json
├── babel.config.json
├── bin/
├── cli.js (900 tokens)
├── biome.json (300 tokens)
├── docs/
├── index.html (400 tokens)
├── style.css (200 tokens)
├── package-lock.json (omitted)
├── package.json (600 tokens)
├── src/
├── index.test.ts (6.9k tokens)
├── index.ts
├── regular/
├── jsonrepair.ts (4.8k tokens)
├── stream.ts
├── streaming/
├── buffer/
├── InputBuffer.test.ts (600 tokens)
├── InputBuffer.ts (400 tokens)
├── OutputBuffer.test.ts (500 tokens)
├── OutputBuffer.ts (700 tokens)
├── core.test.ts (400 tokens)
├── core.ts (5.5k tokens)
├── stack.ts (200 tokens)
├── stream.test.ts (400 tokens)
├── stream.ts (200 tokens)
├── utils/
├── JSONRepairError.ts
├── stringUtils.ts (1100 tokens)
├── test-lib/
├── apps/
├── cjsApp.cjs
├── cjsAppStreaming.cjs
├── esmApp.mjs
├── esmAppStreaming.mjs
├── esmBrowserApp.html (100 tokens)
├── umdApp.cjs
├── umdAppMin.cjs
├── umdBrowserApp.html (100 tokens)
├── cli.test.js (600 tokens)
├── data/
├── invalid.json
├── lib.test.js (400 tokens)
├── output/
├── .gitignore
├── test.html (200 tokens)
├── tools/
├── benchmark/
├── run.mjs (300 tokens)
├── utils/
├── formatTaskResult.mjs (100 tokens)
├── table.mjs (200 tokens)
├── cjs/
├── package.json
├── tsconfig-types.json
├── tsconfig.json (100 tokens)
```
## /.commitlintrc.json
```json path="/.commitlintrc.json"
{
"extends": ["@commitlint/config-conventional"]
}
```
## /.github/workflows/build.yaml
```yaml path="/.github/workflows/build.yaml"
name: Node.js CI
on: [push, pull_request]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [20.x, 22.x, 24.x]
steps:
- uses: actions/checkout@v4
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm run build-and-test
env:
CI: true
```
## /.gitignore
```gitignore path="/.gitignore"
lib
node_modules
.idea
```
## /.husky/commit-msg
```husky/commit-msg path="/.husky/commit-msg"
npx --no -- commitlint --edit ${1}
```
## /.husky/pre-commit
```husky/pre-commit path="/.husky/pre-commit"
# npm test
```
## /CHANGELOG.md
# Changelog
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
## [3.14.0](https://github.com/josdejong/jsonrepair/compare/v3.13.3...v3.14.0) (2026-04-16)
### Features
* [#160](https://github.com/josdejong/jsonrepair/issues/160) repair backslash-escaped newline characters ([3266141](https://github.com/josdejong/jsonrepair/commit/3266141dcb0d25c43008825c1749446bbcc44e9b))
### [3.13.3](https://github.com/josdejong/jsonrepair/compare/v3.13.2...v3.13.3) (2026-03-10)
### Bug Fixes
* [#156](https://github.com/josdejong/jsonrepair/issues/156) repair zero-width spaces ([ca329f7](https://github.com/josdejong/jsonrepair/commit/ca329f724ff79443d66898a40444887047ae65e6))
### [3.13.2](https://github.com/josdejong/jsonrepair/compare/v3.13.1...v3.13.2) (2026-01-14)
### Bug Fixes
* [#150](https://github.com/josdejong/jsonrepair/issues/150) XSS risk when parsing a repaired regex using `eval` ([cc2934b](https://github.com/josdejong/jsonrepair/commit/cc2934bf7b72a244d16cbccf911ac5aa718e9d59))
### [3.13.1](https://github.com/josdejong/jsonrepair/compare/v3.13.0...v3.13.1) (2025-09-19)
### Bug Fixes
* [#146](https://github.com/josdejong/jsonrepair/issues/146) repair fenced code blocks preceded by whitespace ([6fa666e](https://github.com/josdejong/jsonrepair/commit/6fa666e1e917dda38c90cd6858b98742795e638e))
## [3.13.0](https://github.com/josdejong/jsonrepair/compare/v3.12.0...v3.13.0) (2025-07-10)
### Features
* repair invalid fenced Markdown code blocks (fixes [#143](https://github.com/josdejong/jsonrepair/issues/143)) ([92cf61d](https://github.com/josdejong/jsonrepair/commit/92cf61d996d7c86737b92288968e2ff9332c0419))
## [3.12.0](https://github.com/josdejong/jsonrepair/compare/v3.11.2...v3.12.0) (2025-02-06)
### Features
* repair markdown fenced code blocks ([#136](https://github.com/josdejong/jsonrepair/issues/136)) ([#141](https://github.com/josdejong/jsonrepair/issues/141)) ([30cadf7](https://github.com/josdejong/jsonrepair/commit/30cadf799b140a706acbea2746deb8d0b95c87fb))
### [3.11.2](https://github.com/josdejong/jsonrepair/compare/v3.11.1...v3.11.2) (2024-12-20)
### Bug Fixes
* handle repairing a missing comma at a newline ([ecf9588](https://github.com/josdejong/jsonrepair/commit/ecf958896a30f770d165756d391a590154c5745a))
### [3.11.1](https://github.com/josdejong/jsonrepair/compare/v3.11.0...v3.11.1) (2024-12-03)
### Bug Fixes
* [#134](https://github.com/josdejong/jsonrepair/issues/134) broken link to browser bundle in package.json ([4553795](https://github.com/josdejong/jsonrepair/commit/45537959469b2a0a9d7de2b96d6f93b8dfef24aa))
## [3.11.0](https://github.com/josdejong/jsonrepair/compare/v3.10.0...v3.11.0) (2024-11-27)
### Features
* [#133](https://github.com/josdejong/jsonrepair/issues/133) repair the missing end quote of a string value containing commas in an object ([1899f70](https://github.com/josdejong/jsonrepair/commit/1899f7032fb510e791521c369db6821a1f98838d))
## [3.10.0](https://github.com/josdejong/jsonrepair/compare/v3.9.0...v3.10.0) (2024-11-05)
### Features
* repair unquoted urls ([dfca0f6](https://github.com/josdejong/jsonrepair/commit/dfca0f6bd3a722f9aa018f133c8876332565cf84))
## [3.9.0](https://github.com/josdejong/jsonrepair/compare/v3.8.1...v3.9.0) (2024-10-21)
### Features
* [#126](https://github.com/josdejong/jsonrepair/issues/126) [#130](https://github.com/josdejong/jsonrepair/issues/130) repair strings containing a colon or parenthesis ([948c571](https://github.com/josdejong/jsonrepair/commit/948c571698b58741bf0f290cf733eb96c9d9dfc7))
* [#126](https://github.com/josdejong/jsonrepair/issues/126) [#130](https://github.com/josdejong/jsonrepair/issues/130) repair strings containing a colon or parenthesis ([99830b7](https://github.com/josdejong/jsonrepair/commit/99830b700837d856dd3da8f0f87d7cbcd1ea838b))
### [3.8.1](https://github.com/josdejong/jsonrepair/compare/v3.8.0...v3.8.1) (2024-09-18)
### Bug Fixes
* code style adjustments, use template literals consistently ([594f535](https://github.com/josdejong/jsonrepair/commit/594f535531ac14f509f78435407fb1b25d72fed4))
## [3.8.0](https://github.com/josdejong/jsonrepair/compare/v3.7.1...v3.8.0) (2024-05-15)
### Features
* [#125](https://github.com/josdejong/jsonrepair/issues/125) strip leading commas in objects and arrays ([0ee8597](https://github.com/josdejong/jsonrepair/commit/0ee859789d1bfbb1ab2f593367570489ad172569))
* [#127](https://github.com/josdejong/jsonrepair/issues/127) skip ellipsis in arrays and objects ([019e509](https://github.com/josdejong/jsonrepair/commit/019e509576e0e47a7aface746498b8315c0f9489))
* strip ellipsis from empty objects and arrays ([5731996](https://github.com/josdejong/jsonrepair/commit/57319969b165e580e0485f1ae31c307fd7eb5bd8))
### [3.7.1](https://github.com/josdejong/jsonrepair/compare/v3.7.0...v3.7.1) (2024-05-09)
### Bug Fixes
* [#126](https://github.com/josdejong/jsonrepair/issues/126) more robust detection of MongoDB data types and JSONP callbacks ([58fe64c](https://github.com/josdejong/jsonrepair/commit/58fe64ce62d7a3840f427df2dabb1fa748e540de))
## [3.7.0](https://github.com/josdejong/jsonrepair/compare/v3.6.1...v3.7.0) (2024-04-24)
### Features
* turn invalid numbers into strings ([b2c68f3](https://github.com/josdejong/jsonrepair/commit/b2c68f354e9150fc4054df0498871fec4e32a52e))
### Bug Fixes
* [#120](https://github.com/josdejong/jsonrepair/issues/120) turn invalid numbers into strings ([514ef89](https://github.com/josdejong/jsonrepair/commit/514ef89871dc02abdf5428e4603ae2d85a266747))
* [#123](https://github.com/josdejong/jsonrepair/issues/123) repair regular expressions ([c475377](https://github.com/josdejong/jsonrepair/commit/c475377be0d722b1c578314204c488970d4e1746))
### [3.6.1](https://github.com/josdejong/jsonrepair/compare/v3.6.0...v3.6.1) (2024-04-11)
### Bug Fixes
* [#117](https://github.com/josdejong/jsonrepair/issues/117), [#121](https://github.com/josdejong/jsonrepair/issues/121) improve repairing of truncated strings ([0fe1757](https://github.com/josdejong/jsonrepair/commit/0fe1757c62c035f6b8a118b3ba3213597912c923))
## [3.6.0](https://github.com/josdejong/jsonrepair/compare/v3.5.1...v3.6.0) (2024-02-13)
### Features
* repair unescaped double quotes (WIP) ([9e7b04b](https://github.com/josdejong/jsonrepair/commit/9e7b04bf4ba3b3a805e379aab25e55ee729b3884))
### Bug Fixes
* [#102](https://github.com/josdejong/jsonrepair/issues/102) repair truncated string containing a single quote ([bc46250](https://github.com/josdejong/jsonrepair/commit/bc46250e6a990660261958325a7b2e2d8ed83099))
* [#114](https://github.com/josdejong/jsonrepair/issues/114) [#102](https://github.com/josdejong/jsonrepair/issues/102) repair unescaped quotes in a string ([647326c](https://github.com/josdejong/jsonrepair/commit/647326cff40f7b727b357ecf93131f46f8caa7e0))
### [3.5.1](https://github.com/josdejong/jsonrepair/compare/v3.5.0...v3.5.1) (2024-01-10)
### Bug Fixes
* [#112](https://github.com/josdejong/jsonrepair/issues/112) remove comments after a string containing a delimiter ([6e12753](https://github.com/josdejong/jsonrepair/commit/6e12753a144f97b7076e89c7452e68f3b226051f))
## [3.5.0](https://github.com/josdejong/jsonrepair/compare/v3.4.1...v3.5.0) (2023-12-07)
### Features
* [#106](https://github.com/josdejong/jsonrepair/discussions/106) streaming support in Node.js
([#111](https://github.com/josdejong/jsonrepair/pull/111))
### Bug Fixes
* repair a string concat that is not followed by a string
### [3.4.1](https://github.com/josdejong/jsonrepair/compare/v3.4.0...v3.4.1) (2023-11-12)
### Bug Fixes
* [#109](https://github.com/josdejong/jsonrepair/issues/109) fix truncated unicode characters ([c9c8d80](https://github.com/josdejong/jsonrepair/commit/c9c8d80e6f3177442f8a0ddcb1856f206cb0459f))
## [3.4.0](https://github.com/josdejong/jsonrepair/compare/v3.3.0...v3.4.0) (2023-11-01)
### Features
* [#78](https://github.com/josdejong/jsonrepair/issues/78) repair truncated JSON ([17a002a](https://github.com/josdejong/jsonrepair/commit/17a002a55c6f0fdeb6bf064d85d4ff6b03509963))
## [3.3.0](https://github.com/josdejong/jsonrepair/compare/v3.2.4...v3.3.0) (2023-11-01)
### Features
* [#103](https://github.com/josdejong/jsonrepair/issues/103) remove redundant close brackets ([f81ffad](https://github.com/josdejong/jsonrepair/commit/f81ffad5f30fc7e3cec8f1481f8b189c6a4eb49f))
### [3.2.4](https://github.com/josdejong/jsonrepair/compare/v3.2.3...v3.2.4) (2023-10-04)
### Bug Fixes
* [#101](https://github.com/josdejong/jsonrepair/issues/101) implement a smarter way to fix both missing end quotes and unescaped newline characters ([51a4de9](https://github.com/josdejong/jsonrepair/commit/51a4de923d78d23d7fbd39be1810713a7db3eea9))
### [3.2.3](https://github.com/josdejong/jsonrepair/compare/v3.2.2...v3.2.3) (2023-09-27)
### Bug Fixes
* [#99](https://github.com/josdejong/jsonrepair/issues/99) fix repairing single quoted strings containing quotes such as backtick ([b4b9180](https://github.com/josdejong/jsonrepair/commit/b4b918017991f783d98ed376792ec97df74c678d))
* repair numeric values with trailing zeros like `00789` by changing them into a string ([399f593](https://github.com/josdejong/jsonrepair/commit/399f593d110c06172b7eddf5b7d4cc9f0cd6969e))
### [3.2.2](https://github.com/josdejong/jsonrepair/compare/v3.2.1...v3.2.2) (2023-09-22)
### Bug Fixes
* [#100](https://github.com/josdejong/jsonrepair/issues/100) jsonrepair sometimes crashing when repairing missing quotes (regression since v3.2.1) ([f573da2](https://github.com/josdejong/jsonrepair/commit/f573da2f0575c0434dface16fe906754dc47f124))
### [3.2.1](https://github.com/josdejong/jsonrepair/compare/v3.2.0...v3.2.1) (2023-09-20)
### Bug Fixes
* [#97](https://github.com/josdejong/jsonrepair/issues/97) improved handling of missing start and end quotes ([82df750](https://github.com/josdejong/jsonrepair/commit/82df75049ffd4aecc275bedb8f594a462027a834))
* [#98](https://github.com/josdejong/jsonrepair/issues/98) wrong position reported in the error message of invalid numbers ([5093616](https://github.com/josdejong/jsonrepair/commit/5093616f91454cafa47d482e830017781155cd79))
* throw an error on numbers with a leading zero instead of splitting them in two ([829d3ee](https://github.com/josdejong/jsonrepair/commit/829d3eebb02a1d5bbf395ba3fb7d5a4814fd1b3a))
## [3.2.0](https://github.com/josdejong/jsonrepair/compare/v3.1.0...v3.2.0) (2023-06-13)
### Features
* repair a missing object value ([2cd756f](https://github.com/josdejong/jsonrepair/commit/2cd756f7806320003551b1fea63e2495dba39080))
### Bug Fixes
* [#93](https://github.com/josdejong/jsonrepair/issues/93) repair `undefined` values by replacing them with `null` ([af348d7](https://github.com/josdejong/jsonrepair/commit/af348d723586bc06f93a510d30154bd484052165))
## [3.1.0](https://github.com/josdejong/jsonrepair/compare/v3.0.3...v3.1.0) (2023-05-04)
### Features
* fix broken numbers at the end of the string ([c42d9dd](https://github.com/josdejong/jsonrepair/commit/c42d9dd9ac6c60f2ebef8292d0485409f90d2ab9))
* fix broken numbers at the end of the string ([#91](https://github.com/josdejong/jsonrepair/issues/91)) ([9ad00fd](https://github.com/josdejong/jsonrepair/commit/9ad00fd09b600ac191bda9dec4469aa553e97645))
### [3.0.3](https://github.com/josdejong/jsonrepair/compare/v3.0.2...v3.0.3) (2023-04-17)
### Bug Fixes
* [#89](https://github.com/josdejong/jsonrepair/issues/89) wrongly parsing strings that contain a double quote left or right ([4023ece](https://github.com/josdejong/jsonrepair/commit/4023ece4442a85bc615e936fbb1de8683e821e91))
### [3.0.2](https://github.com/josdejong/jsonrepair/compare/v3.0.1...v3.0.2) (2023-01-06)
### Bug Fixes
* error handling unicode characters containing a `9` ([d665ec2](https://github.com/josdejong/jsonrepair/commit/d665ec2f934f8d499f0498d5a1a91515ee4dbd72))
### [3.0.1](https://github.com/josdejong/jsonrepair/compare/v3.0.0...v3.0.1) (2022-12-20)
### Bug Fixes
* improve resolving unquoted strings and missing colon ([45cd4e4](https://github.com/josdejong/jsonrepair/commit/45cd4e45c6c10fac148cf6a037752586ed4fb2d5))
## 2021-12-19, version 3.0.0
- Complete rewrite of the parser in TypeScript, with improved performance.
- Can repair some additional cases of broken JSON.
⚠ BREAKING CHANGES
- Changed the API from default export `import jsonrepair from 'jsonrepair'` to named export `import { jsonrepair} from 'jsonrepair'`
- Changed in UMD export from `jsonrepair` to `JSONRepair.jsonrepair`
- Changed the error class to `JSONRepairError` with a property `.position`
## 2021-06-09, version 2.2.1
- Improved handling of trailing commas.
- Improved handling of newline delimited JSON containing commas.
- Improved handling of repairing objects/arrays with missing closing bracket.
## 2021-04-01, version 2.2.0
- Implement #16: turn an escaped string containing JSON into valid JSON.
## 2021-04-01, version 2.1.0
- Implemented command line interface (CLI), see #34.
## 2021-03-01, version 2.0.1
- Performance improvements.
## 2021-01-13, version 2.0.0
- Renamed the library from `simple-json-repair` to `jsonrepair`.
Thanks a lot @vmx for making this npm package name available!
- Change source code from TypeScript to JavaScript. Reasons: TypeScript
conflicts with using native ESM, requiring ugly workarounds.
Due to some (old) TypeScript issues we also have to use `@ts-ignore` a lot.
Using TypeScript makes running tests slower. And in this case, TypeScript
hardly adds value since we have a very simple API and function signatures.
## 2020-11-06, version 1.1.0
- Implement support for string concatenation.
- Implement support for adding missing end brackets for objects and arrays.
## 2020-11-05, version 1.0.1
- Fixed ESM and UMD builds missing in npm package.
## 2020-11-05, version 1.0.0
- Initial release, code extracted from the library `jsoneditor`.
## /LICENSE.md
The ISC License
Copyright (c) 2020-2026 by Jos de Jong
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
## /README.md
# jsonrepair
Repair invalid JSON documents.
Try it out in a minimal demo: https://josdejong.github.io/jsonrepair/
Use it in a full-fledged application: https://jsoneditoronline.org
Read the background article ["How to fix JSON and validate it with ease"](https://jsoneditoronline.org/indepth/parse/fix-json/)
The following issues can be fixed:
- Add missing quotes around keys
- Add missing escape characters
- Add missing commas
- Add missing closing brackets
- Repair truncated JSON
- Replace single quotes with double quotes
- Replace special quote characters like `“...”` with regular double quotes
- Replace special white space characters with regular spaces
- Replace Python constants `None`, `True`, and `False` with `null`, `true`, and `false`
- Strip trailing commas
- Strip comments like `/* ... */` and `// ...`
- Strip fenced code blocks like `` ```json`` and `` ``` ``
- Strip ellipsis in arrays and objects like `[1, 2, 3, ...]`
- Strip JSONP notation like `callback({ ... })`
- Strip escape characters from an escaped string like `{\"stringified\": \"content\"}`
- Strip MongoDB data types like `NumberLong(2)` and `ISODate("2012-12-19T06:01:17.171Z")`
- Concatenate strings like `"long text" + "more text on next line"`
- Turn newline delimited JSON into a valid JSON array, for example:
```
{ "id": 1, "name": "John" }
{ "id": 2, "name": "Sarah" }
```
The `jsonrepair` library has streaming support and can handle infinitely large documents.
## Install
```
$ npm install jsonrepair
```
Note that in the `lib` folder, there are builds for ESM, UMD, and CommonJs.
## Use
### ES module
Use the `jsonrepair` function using an ES modules import:
```js
import { jsonrepair } from 'jsonrepair'
try {
// The following is invalid JSON: is consists of JSON contents copied from
// a JavaScript code base, where the keys are missing double quotes,
// and strings are using single quotes:
const json = "{name: 'John'}"
const repaired = jsonrepair(json)
console.log(repaired) // '{"name": "John"}'
} catch (err) {
console.error(err)
}
```
### Streaming API
Use the streaming API in Node.js:
```js
import { createReadStream, createWriteStream } from 'node:fs'
import { pipeline } from 'node:stream'
import { jsonrepairTransform } from 'jsonrepair/stream'
const inputStream = createReadStream('./data/broken.json')
const outputStream = createWriteStream('./data/repaired.json')
pipeline(inputStream, jsonrepairTransform(), outputStream, (err) => {
if (err) {
console.error(err)
} else {
console.log('done')
}
})
// or using .pipe() instead of pipeline():
// inputStream
// .pipe(jsonrepairTransform())
// .pipe(outputStream)
// .on('error', (err) => console.error(err))
// .on('finish', () => console.log('done'))
```
### CommonJS
Use in CommonJS (not recommended):
```js
const { jsonrepair } = require('jsonrepair')
const json = "{name: 'John'}"
console.log(jsonrepair(json)) // '{"name": "John"}'
```
### UMD
Use with UMD in the browser (not recommended):
```html
<script src="/node_modules/jsonrepair/lib/umd/jsonrepair.js"></script>
<script>
const { jsonrepair } = JSONRepair
const json = "{name: 'John'}"
console.log(jsonrepair(json)) // '{"name": "John"}'
</script>
```
### Python
Use in Python via [`PythonMonkey`](https://github.com/Distributive-Network/PythonMonkey#pythonmonkey).
1. Install `jsonrepair` via `npm install jsonrepair`
2. Install `PythonMonkey` via `pip install pythonmonkey`
3. Use the libraries in a Python script:
```python
import pythonmonkey
jsonrepair = pythonmonkey.require('jsonrepair').jsonrepair
json = "[1,2,3,"
repaired = jsonrepair(json)
print(repaired)
# [1,2,3]
```
## API
### Regular API
You can use `jsonrepair` as a function or as a streaming transform. Broken JSON is passed to the function, and the function either returns the repaired JSON, or throws an `JSONRepairError` exception when an issue is encountered which could not be solved.
```ts
// @throws JSONRepairError
jsonrepair(json: string) : string
```
### Streaming API
The streaming API is availabe in `jsonrepair/stream` and can be used in a [Node.js stream](https://nodejs.org/api/stream.html). It consists of a transform function that can be used in a stream pipeline.
```ts
jsonrepairTransform(options?: { chunkSize?: number, bufferSize?: number }) : Transform
```
The option `chunkSize` determines the size of the chunks that the transform outputs, and is `65536` bytes by default. Changing `chunkSize` can influcence the performance.
The option `bufferSize` determines how many bytes of the input and output stream are kept in memory and is also `65536` bytes by default. This buffer is used as a "moving window" on the input and output. This is necessary because `jsonrepair` must look ahead or look back to see what to fix, and it must sometimes walk back the generated output to insert a missing comma for example. The `bufferSize` must be larger than the length of the largest string and whitespace in the JSON data, otherwise, and error is thrown when processing the data. Making `bufferSize` very large will result in more memory usage and less performance.
## Command Line Interface (CLI)
When `jsonrepair` is installed globally using npm, it can be used on the command line. To install `jsonrepair` globally:
```bash
$ npm install -g jsonrepair
```
Usage:
```
$ jsonrepair [filename] {OPTIONS}
```
Options:
```
--version, -v Show application version
--help, -h Show this message
--output, -o Output file
--overwrite Overwrite the input file
--buffer Buffer size in bytes, for example 64K (default) or 1M
```
Example usage:
```
$ jsonrepair broken.json # Repair a file, output to console
$ jsonrepair broken.json > repaired.json # Repair a file, output to file
$ jsonrepair broken.json --output repaired.json # Repair a file, output to file
$ jsonrepair broken.json --overwrite # Repair a file, replace the file itself
$ cat broken.json | jsonrepair # Repair data from an input stream
$ cat broken.json | jsonrepair > repaired.json # Repair data from an input stream, output to file
```
## Alternatives:
Similar libraries:
- https://github.com/RyanMarcus/dirty-json
## Develop
When implementing a fix or a new feature, it important to know that there are currently two implementations:
- `src/regular` This is a non-streaming implementation. The code is small and works for files up to 512MB, ideal for usage in the browser.
- `src/streaming` A streaming implementation that can be used in Node.js. The code is larger and more complex, and the implementation uses a configurable `bufferSize` and `chunkSize`. When the parsed document contains a string or number that is longer than the configured `bufferSize`, the library will throw an "Index out of range" error since it cannot hold the full string in the buffer. When configured with an infinite buffer size, the streaming implementation works the same as the regular implementation. In that case this out of range error cannot occur, but it makes the performance worse and the application can run out of memory when repairing large documents.
Both implementations are tested against the same suite of unit tests in `src/index.test.ts`.
Scripts:
Script | Description
---------- | -----------
`npm install` | Install the dependencies once
`npm run build` | Build the library (ESM, CommonJs, and UMD output in the folder `lib`)
`npm test` | Run the unit tests
`npm run lint` | Run the linter (eslint)
`npm run format` | Automatically fix linter issues
`npm run build-and-test` | Run the linter, build all, and run unit tests and integration tests
`npm run release` | Release a new version. This will lint, test, build, increment the version number, push the changes to git, add a git version tag, and publish the npm package.
`npm run release-dry-run` | Run all release steps and see the change list without actually publishing:
## License
Released under the [ISC license](LICENSE.md).
## /babel-cjs.config.json
```json path="/babel-cjs.config.json"
{
"presets": [
[
"@babel/env",
{
"targets": "> 0.25%, not dead"
}
],
["@babel/preset-typescript"]
],
"ignore": ["src/**/*.test.ts"]
}
```
## /babel.config.json
```json path="/babel.config.json"
{
"presets": [
[
"@babel/env",
{
"targets": {
"browsers": "> 0.25%, not dead"
},
"modules": false
}
],
["@babel/preset-typescript"]
],
"ignore": ["src/**/*.test.ts"]
}
```
## /bin/cli.js
```js path="/bin/cli.js"
#!/usr/bin/env node
import { createReadStream, createWriteStream, readFileSync, renameSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { pipeline as pipelineCallback } from 'node:stream'
import { fileURLToPath } from 'node:url'
import { promisify } from 'node:util'
import { jsonrepairTransform } from '../lib/esm/stream.js'
const pipeline = promisify(pipelineCallback)
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
function processArgs(args) {
const options = {
version: false,
help: false,
overwrite: false,
bufferSize: undefined,
inputFile: null,
outputFile: null
}
// we skip the first two args, since they contain node and the script path
let i = 2
while (i < args.length) {
const arg = args[i]
switch (arg) {
case '-v':
case '--version':
options.version = true
break
case '-h':
case '--help':
options.help = true
break
case '--overwrite':
options.overwrite = true
break
case '--buffer':
i++
options.bufferSize = parseSize(args[i])
break
case '-o':
case '--output':
i++
options.outputFile = args[i]
break
default:
if (options.inputFile == null) {
options.inputFile = arg
} else {
throw new Error(`Unexpected argument "${arg}"`)
}
}
i++
}
return options
}
async function run(options) {
if (options.version) {
outputVersion()
return
}
if (options.help) {
outputHelp()
return
}
if (options.overwrite) {
if (!options.inputFile) {
console.error('Error: cannot use --overwrite: no input file provided')
process.exit(1)
}
if (options.outputFile) {
console.error('Error: cannot use --overwrite: there is also an --output provided')
process.exit(1)
}
const dateStr = new Date().toISOString().replace(/\W/g, '-')
const tempFileSuffix = `.repair-${dateStr}.json`
const tempFile = options.inputFile + tempFileSuffix
try {
const readStream = createReadStream(options.inputFile)
const writeStream = createWriteStream(tempFile)
await pipeline(
readStream,
jsonrepairTransform({ bufferSize: options.bufferSize }),
writeStream
)
renameSync(tempFile, options.inputFile)
} catch (err) {
process.stderr.write(err.toString())
process.exit(1)
}
return
}
try {
const readStream = options.inputFile ? createReadStream(options.inputFile) : process.stdin
const writeStream = options.outputFile ? createWriteStream(options.outputFile) : process.stdout
await pipeline(readStream, jsonrepairTransform({ bufferSize: options.bufferSize }), writeStream)
} catch (err) {
process.stderr.write(err.toString())
process.exit(1)
}
}
function outputVersion() {
const file = join(__dirname, '../package.json')
const pkg = JSON.parse(String(readFileSync(file, 'utf-8')))
console.log(pkg.version)
}
function parseSize(size) {
// match
const match = size.match(/^(\d+)([KMG]?)$/)
if (!match) {
throw new Error(`Buffer size "${size}" not recognized. Examples: 65536, 512K, 2M`)
}
const num = Number.parseInt(match[1], 10)
const suffix = match[2] // K, M, or G
switch (suffix) {
case 'K':
return num * 1024
case 'M':
return num * 1024 * 1024
case 'G':
return num * 1024 * 1024 * 1024
default:
return num
}
}
const help = `
jsonrepair
https://github.com/josdejong/jsonrepair
Repair invalid JSON documents. When a document could not be repaired, the output will be left unchanged.
Usage:
jsonrepair [filename] {OPTIONS}
Options:
--version, -v Show application version
--help, -h Show this message
--output, -o Output file
--overwrite Overwrite the input file
--buffer Buffer size in bytes, for example 64K (default) or 1M
Example usage:
jsonrepair broken.json # Repair a file, output to console
jsonrepair broken.json > repaired.json # Repair a file, output to file
jsonrepair broken.json --output repaired.json # Repair a file, output to file
jsonrepair broken.json --overwrite # Repair a file, replace the file itself
cat broken.json | jsonrepair # Repair data from an input stream
cat broken.json | jsonrepair > repaired.json # Repair data from an input stream, output to file
`
function outputHelp() {
console.log(help)
}
const options = processArgs(process.argv)
await run(options)
```
## /biome.json
```json path="/biome.json"
{
"$schema": "https://biomejs.dev/schemas/2.0.0-beta.6/schema.json",
"vcs": { "enabled": false, "clientKind": "git", "useIgnoreFile": false },
"files": {
"ignoreUnknown": false,
"includes": ["**", "!**/lib", "!**/test-lib/data", "!**/test-lib/output", "!**/package.json"]
},
"formatter": {
"enabled": true,
"useEditorconfig": true,
"formatWithErrors": false,
"indentStyle": "space",
"indentWidth": 2,
"lineEnding": "lf",
"lineWidth": 100,
"attributePosition": "auto",
"bracketSpacing": true
},
"assist": { "actions": { "source": { "organizeImports": "on" } } },
"linter": {
"enabled": true,
"rules": {
"recommended": true,
"style": {
"noParameterAssign": "error",
"useAsConstAssertion": "error",
"useDefaultParameterLast": "error",
"useEnumInitializers": "error",
"useSelfClosingElements": "error",
"useSingleVarDeclarator": "error",
"noUnusedTemplateLiteral": "error",
"useNumberNamespace": "error",
"noInferrableTypes": "error",
"noUselessElse": "error"
}
}
},
"javascript": {
"formatter": {
"jsxQuoteStyle": "double",
"quoteProperties": "asNeeded",
"trailingCommas": "none",
"semicolons": "asNeeded",
"arrowParentheses": "always",
"bracketSameLine": false,
"quoteStyle": "single",
"attributePosition": "auto",
"bracketSpacing": true
}
}
}
```
## /docs/index.html
```html path="/docs/index.html"
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>jsonrepair playground</title>
<link rel="stylesheet" href="style.css">
<meta name="description" content="Repair broken JSON documents">
<meta name="keywords" content="json, repair, simple, json-simple-repair, fix, invalid">
<meta name="viewport" content="width=device-width, initial-scale=1" />
</head>
<body>
<div class="app">
<h1>jsonrepair</h1>
<div class="info">
Repair invalid JSON documents. Documentation:
<a href="https://github.com/josdejong/jsonrepair">
https://github.com/josdejong/jsonrepair
</a>
</div>
<div class="playground">
<div class="column">
<label for="input-text">Input (invalid JSON)</label>
<textarea id="input-text" autocomplete="off" autocapitalize="off" spellcheck="false">{
'firstName': 'John'
lastName: “Smith”
fullName: John Smith,
// TODO: fill in last season
scores: [ 7.8 6.3 7.1, ],
"about": "John loves a challenge, " +
"but can quickly lose focus."
</textarea>
</div>
<div class="column">
<label for="output-text">Output (fixed JSON)</label>
<textarea id="output-text" readonly>loading...</textarea>
</div>
</div>
</div>
<script type="module">
import { jsonrepair } from 'https://cdn.jsdelivr.net/npm/jsonrepair@latest/+esm'
window.jsonrepair = jsonrepair
console.info("Hi there! You can use jsonrepair on the command line")
function repairIt () {
try {
outputText.value = jsonrepair(inputText.value)
outputText.classList.remove('error')
} catch (err) {
outputText.value = err.message
outputText.classList.add('error')
}
}
const inputText = document.getElementById('input-text')
const outputText = document.getElementById('output-text')
inputText.addEventListener('input', repairIt)
repairIt()
</script>
</body>
</html>
```
## /docs/style.css
```css path="/docs/style.css"
html,
body {
width: 100%;
height: 100%;
margin: 0;
padding: 0;
font-family: ubuntu, arial, sans-serif;
overflow: hidden;
}
body,
textarea {
color: #4d4d4d;
}
a {
color: dodgerblue;
}
a:hover {
color: #187ad7;
}
body {
display: flex;
justify-content: center;
}
h1 {
font-size: 150%;
color: dodgerblue;
margin: 0;
padding: 20px 0;
}
.app {
max-width: 1000px;
flex: 1;
padding: 10px;
display: flex;
flex-direction: column;
overflow: auto;
}
.info {
padding: 0 0 40px 0;
}
.playground {
flex: 1;
display: flex;
gap: 20px;
}
@media (max-width: 650px) {
.playground {
flex-direction: column;
}
}
.column {
flex: 1;
display: flex;
flex-direction: column;
}
.separator {
width: 20px;
}
label {
font-weight: bold;
margin-bottom: 10px;
}
textarea {
flex: 1;
border: 1px solid #bfbfbf;
border-radius: 3px;
padding: 5px;
box-sizing: border-box;
font-family: monospace;
max-height: 500px;
min-height: 150px;
resize: none;
}
textarea[readonly] {
background: #f5f5f5;
}
.error {
color: #f65252;
}
```
## /package.json
```json path="/package.json"
{
"name": "jsonrepair",
"version": "3.14.0",
"description": "Repair broken JSON documents",
"repository": {
"type": "git",
"url": "https://github.com/josdejong/jsonrepair.git"
},
"type": "module",
"main": "lib/cjs/index.js",
"module": "lib/esm/index.js",
"browser": "lib/umd/jsonrepair.min.js",
"types": "lib/types/index.d.ts",
"sideEffects": false,
"exports": {
".": {
"import": "./lib/esm/index.js",
"require": "./lib/cjs/index.js",
"types": "./lib/types/index.d.ts"
},
"./stream": {
"import": "./lib/esm/stream.js",
"require": "./lib/cjs/stream.js",
"types": "./lib/types/stream.d.ts"
}
},
"keywords": [
"simple",
"json",
"repair",
"fix",
"invalid",
"stream",
"streaming"
],
"bin": {
"jsonrepair": "./bin/cli.js"
},
"scripts": {
"test": "vitest watch src",
"test:it": "vitest run src",
"build": "npm-run-all build:**",
"build:clean": "del-cli lib",
"build:esm": "babel src --out-dir lib/esm --extensions \".ts\" --source-maps --config-file ./babel.config.json",
"build:cjs": "babel src --out-dir lib/cjs --extensions \".ts\" --source-maps --config-file ./babel-cjs.config.json && cpy tools/cjs/package.json lib/cjs --flat",
"build:umd": "rollup lib/esm/index.js --format umd --name JSONRepair --sourcemap --output.file lib/umd/jsonrepair.js && cpy tools/cjs/package.json lib/umd --flat",
"build:umd:min": "uglifyjs --compress --mangle --source-map --comments --output lib/umd/jsonrepair.min.js -- lib/umd/jsonrepair.js",
"build:types": "tsc --project tsconfig-types.json",
"build:validate": "vitest run test-lib",
"lint": "biome check",
"format": "biome check --write",
"benchmark": "npm run build:esm && node tools/benchmark/run.mjs",
"build-and-test": "npm run lint && npm run test:it && npm run build",
"release": "npm-run-all release:**",
"release:build-and-test": "npm run build-and-test",
"release:version": "standard-version",
"release:push": "git push && git push --tag",
"release:publish": "npm publish",
"release-dry-run": "npm run build-and-test && standard-version --dry-run",
"prepare": "husky"
},
"files": [
"README.md",
"LICENSE.md",
"lib"
],
"author": "Jos de Jong",
"license": "ISC",
"devDependencies": {
"@babel/cli": "7.28.6",
"@babel/core": "7.29.0",
"@babel/plugin-transform-typescript": "7.28.6",
"@babel/preset-env": "7.29.2",
"@babel/preset-typescript": "7.28.5",
"@biomejs/biome": "2.4.12",
"@commitlint/cli": "20.5.0",
"@commitlint/config-conventional": "20.5.0",
"@types/node": "25.6.0",
"cpy-cli": "7.0.0",
"del-cli": "7.0.0",
"husky": "9.1.7",
"npm-run-all": "4.1.5",
"rollup": "4.60.1",
"standard-version": "9.5.0",
"tinybench": "6.0.0",
"ts-node": "10.9.2",
"typescript": "5.9.3",
"uglify-js": "3.19.3",
"vitest": "4.1.4"
}
}
```
## /src/index.test.ts
```ts path="/src/index.test.ts"
import { describe, expect, test } from 'vitest'
import { jsonrepair as jsonRepairRegular } from './index'
import { jsonrepairCore } from './streaming/core'
import { JSONRepairError } from './utils/JSONRepairError'
const implementations = [
{ name: 'regular', jsonrepair: jsonRepairRegular },
{ name: 'streaming', jsonrepair: createStreamingRepairWrapper() }
]
describe.each(implementations)('jsonrepair [$name]', ({ jsonrepair }) => {
describe('parse valid JSON', () => {
test('parse full JSON object', () => {
const text = '{"a":2.3e100,"b":"str","c":null,"d":false,"e":[1,2,3]}'
const parsed = jsonrepair(text)
expect(parsed).toBe(text)
})
test('parse whitespace', () => {
assertRepair(' { \n } \t ')
})
test('parse object', () => {
assertRepair('{}')
assertRepair('{ }')
assertRepair('{"a": {}}')
assertRepair('{"a": "b"}')
assertRepair('{"a": 2}')
})
test('parse array', () => {
assertRepair('[]')
assertRepair('[ ]')
assertRepair('[1,2,3]')
assertRepair('[ 1 , 2 , 3 ]')
assertRepair('[1,2,[3,4,5]]')
assertRepair('[{}]')
assertRepair('{"a":[]}')
assertRepair('[1, "hi", true, false, null, {}, []]')
})
test('parse number', () => {
assertRepair('23')
assertRepair('0')
assertRepair('0e+2')
assertRepair('0.0')
assertRepair('-0')
assertRepair('2.3')
assertRepair('2300e3')
assertRepair('2300e+3')
assertRepair('2300e-3')
assertRepair('-2')
assertRepair('2e-3')
assertRepair('2.3e-3')
})
test('parse string', () => {
assertRepair('"str"')
assertRepair('"\\"\\\\\\/\\b\\f\\n\\r\\t"')
assertRepair('"\\u260E"')
})
test('parse keywords', () => {
assertRepair('true')
assertRepair('false')
assertRepair('null')
})
test('correctly handle strings equaling a JSON delimiter', () => {
assertRepair('""')
assertRepair('"["')
assertRepair('"]"')
assertRepair('"{"')
assertRepair('"}"')
assertRepair('":"')
assertRepair('","')
})
test('supports unicode characters in a string', () => {
expect(jsonrepair('"★"')).toBe('"★"')
expect(jsonrepair('"\u2605"')).toBe('"\u2605"')
expect(jsonrepair('"😀"')).toBe('"😀"')
expect(jsonrepair('"\ud83d\ude00"')).toBe('"\ud83d\ude00"')
expect(jsonrepair('"йнформация"')).toBe('"йнформация"')
})
test('supports escaped unicode characters in a string', () => {
expect(jsonrepair('"\\u2605"')).toBe('"\\u2605"')
expect(jsonrepair('"\\u2605A"')).toBe('"\\u2605A"')
expect(jsonrepair('"\\ud83d\\ude00"')).toBe('"\\ud83d\\ude00"')
expect(
jsonrepair('"\\u0439\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u044f"')
).toBe('"\\u0439\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u044f"')
})
test('supports unicode characters in a key', () => {
expect(jsonrepair('{"★":true}')).toBe('{"★":true}')
expect(jsonrepair('{"\u2605":true}')).toBe('{"\u2605":true}')
expect(jsonrepair('{"😀":true}')).toBe('{"😀":true}')
expect(jsonrepair('{"\ud83d\ude00":true}')).toBe('{"\ud83d\ude00":true}')
})
})
describe('repair invalid JSON', () => {
test('should add missing quotes', () => {
expect(jsonrepair('abc')).toBe('"abc"')
expect(jsonrepair('hello world')).toBe('"hello world"')
expect(jsonrepair('{\nmessage: hello world\n}')).toBe('{\n"message": "hello world"\n}')
expect(jsonrepair('{a:2}')).toBe('{"a":2}')
expect(jsonrepair('{a: 2}')).toBe('{"a": 2}')
expect(jsonrepair('{2: 2}')).toBe('{"2": 2}')
expect(jsonrepair('{true: 2}')).toBe('{"true": 2}')
expect(jsonrepair('{\n a: 2\n}')).toBe('{\n "a": 2\n}')
expect(jsonrepair('[a,b]')).toBe('["a","b"]')
expect(jsonrepair('[\na,\nb\n]')).toBe('[\n"a",\n"b"\n]')
})
test('should repair an unquoted url', () => {
expect(jsonrepair('https://www.bible.com/')).toBe('"https://www.bible.com/"')
expect(jsonrepair('{url:https://www.bible.com/}')).toBe('{"url":"https://www.bible.com/"}')
expect(jsonrepair('{url:https://www.bible.com/,"id":2}')).toBe(
'{"url":"https://www.bible.com/","id":2}'
)
expect(jsonrepair('[https://www.bible.com/]')).toBe('["https://www.bible.com/"]')
expect(jsonrepair('[https://www.bible.com/,2]')).toBe('["https://www.bible.com/",2]')
})
test('should repair an url with missing end quote', () => {
expect(jsonrepair('"https://www.bible.com/')).toBe('"https://www.bible.com/"')
expect(jsonrepair('{"url":"https://www.bible.com/}')).toBe('{"url":"https://www.bible.com/"}')
expect(jsonrepair('{"url":"https://www.bible.com/,"id":2}')).toBe(
'{"url":"https://www.bible.com/","id":2}'
)
expect(jsonrepair('["https://www.bible.com/]')).toBe('["https://www.bible.com/"]')
expect(jsonrepair('["https://www.bible.com/,2]')).toBe('["https://www.bible.com/",2]')
})
test('should add missing end quote', () => {
expect(jsonrepair('"abc')).toBe('"abc"')
expect(jsonrepair("'abc")).toBe('"abc"')
expect(jsonrepair('"12:20')).toBe('"12:20"')
expect(jsonrepair('{"time":"12:20}')).toBe('{"time":"12:20"}')
expect(jsonrepair('{"date":2024-10-18T18:35:22.229Z}')).toBe(
'{"date":"2024-10-18T18:35:22.229Z"}'
)
expect(jsonrepair('"She said:')).toBe('"She said:"')
expect(jsonrepair('{"text": "She said:')).toBe('{"text": "She said:"}')
expect(jsonrepair('["hello, world]')).toBe('["hello", "world"]')
expect(jsonrepair('["hello,"world"]')).toBe('["hello","world"]')
expect(jsonrepair('{"a":"b}')).toBe('{"a":"b"}')
expect(jsonrepair('{"a":"b,"c":"d"}')).toBe('{"a":"b","c":"d"}')
expect(jsonrepair('{"a":"b,"c":"d"}')).toBe('{"a":"b","c":"d"}')
expect(jsonrepair('{"a":"b,c,"d":"e"}')).toBe('{"a":"b,c","d":"e"}')
expect(jsonrepair('{a:"b,c,"d":"e"}')).toBe('{"a":"b,c","d":"e"}')
// expect(jsonrepair('{a:"b,c,}')).toBe('{"a":"b,c"}') // TODO: support this case
expect(jsonrepair('["b,c,]')).toBe('["b","c"]')
expect(jsonrepair('\u2018abc')).toBe('"abc"')
expect(jsonrepair('"it\'s working')).toBe('"it\'s working"')
expect(jsonrepair('["abc+/*comment*/"def"]')).toBe('["abcdef"]')
expect(jsonrepair('["abc/*comment*/+"def"]')).toBe('["abcdef"]')
expect(jsonrepair('["abc,/*comment*/"def"]')).toBe('["abc","def"]')
})
test('should repair truncated JSON', () => {
expect(jsonrepair('"foo')).toBe('"foo"')
expect(jsonrepair('[')).toBe('[]')
expect(jsonrepair('["foo')).toBe('["foo"]')
expect(jsonrepair('["foo"')).toBe('["foo"]')
expect(jsonrepair('["foo",')).toBe('["foo"]')
expect(jsonrepair('{"foo":"bar"')).toBe('{"foo":"bar"}')
expect(jsonrepair('{"foo":"bar')).toBe('{"foo":"bar"}')
expect(jsonrepair('{"foo":')).toBe('{"foo":null}')
expect(jsonrepair('{"foo"')).toBe('{"foo":null}')
expect(jsonrepair('{"foo')).toBe('{"foo":null}')
expect(jsonrepair('{')).toBe('{}')
expect(jsonrepair('2.')).toBe('2.0')
expect(jsonrepair('2e')).toBe('2e0')
expect(jsonrepair('2e+')).toBe('2e+0')
expect(jsonrepair('2e-')).toBe('2e-0')
expect(jsonrepair('{"foo":"bar\\u20')).toBe('{"foo":"bar"}')
expect(jsonrepair('"\\u')).toBe('""')
expect(jsonrepair('"\\u2')).toBe('""')
expect(jsonrepair('"\\u260')).toBe('""')
expect(jsonrepair('"\\u2605')).toBe('"\\u2605"')
expect(jsonrepair('{"s \\ud')).toBe('{"s": null}')
expect(jsonrepair('{"message": "it\'s working')).toBe('{"message": "it\'s working"}')
expect(jsonrepair('{"text":"Hello Sergey,I hop')).toBe('{"text":"Hello Sergey,I hop"}')
expect(jsonrepair('{"message": "with, multiple, commma\'s, you see?')).toBe(
'{"message": "with, multiple, commma\'s, you see?"}'
)
})
test('should repair ellipsis in an array', () => {
expect(jsonrepair('[1,2,3,...]')).toBe('[1,2,3]')
expect(jsonrepair('[1, 2, 3, ... ]')).toBe('[1, 2, 3 ]')
expect(jsonrepair('[1,2,3,/*comment1*/.../*comment2*/]')).toBe('[1,2,3]')
expect(jsonrepair('[\n 1,\n 2,\n 3,\n /*comment1*/ .../*comment2*/\n]')).toBe(
'[\n 1,\n 2,\n 3\n \n]'
)
expect(jsonrepair('{"array":[1,2,3,...]}')).toBe('{"array":[1,2,3]}')
expect(jsonrepair('[1,2,3,...,9]')).toBe('[1,2,3,9]')
expect(jsonrepair('[...,7,8,9]')).toBe('[7,8,9]')
expect(jsonrepair('[..., 7,8,9]')).toBe('[ 7,8,9]')
expect(jsonrepair('[...]')).toBe('[]')
expect(jsonrepair('[ ... ]')).toBe('[ ]')
})
test('should repair ellipsis in an object', () => {
expect(jsonrepair('{"a":2,"b":3,...}')).toBe('{"a":2,"b":3}')
expect(jsonrepair('{"a":2,"b":3,/*comment1*/.../*comment2*/}')).toBe('{"a":2,"b":3}')
expect(jsonrepair('{\n "a":2,\n "b":3,\n /*comment1*/.../*comment2*/\n}')).toBe(
'{\n "a":2,\n "b":3\n \n}'
)
expect(jsonrepair('{"a":2,"b":3, ... }')).toBe('{"a":2,"b":3 }')
expect(jsonrepair('{"nested":{"a":2,"b":3, ... }}')).toBe('{"nested":{"a":2,"b":3 }}')
expect(jsonrepair('{"a":2,"b":3,...,"z":26}')).toBe('{"a":2,"b":3,"z":26}')
expect(jsonrepair('{"a":2,"b":3,...}')).toBe('{"a":2,"b":3}')
expect(jsonrepair('{...}')).toBe('{}')
expect(jsonrepair('{ ... }')).toBe('{ }')
})
test('should add missing start quote', () => {
expect(jsonrepair('abc"')).toBe('"abc"')
expect(jsonrepair('[a","b"]')).toBe('["a","b"]')
expect(jsonrepair('[a",b"]')).toBe('["a","b"]')
expect(jsonrepair('{"a":"foo","b":"bar"}')).toBe('{"a":"foo","b":"bar"}')
expect(jsonrepair('{a":"foo","b":"bar"}')).toBe('{"a":"foo","b":"bar"}')
expect(jsonrepair('{"a":"foo",b":"bar"}')).toBe('{"a":"foo","b":"bar"}')
expect(jsonrepair('{"a":foo","b":"bar"}')).toBe('{"a":"foo","b":"bar"}')
})
test('should stop at the first next return when missing an end quote', () => {
expect(jsonrepair('[\n"abc,\n"def"\n]')).toBe('[\n"abc",\n"def"\n]')
expect(jsonrepair('[\n"abc, \n"def"\n]')).toBe('[\n"abc", \n"def"\n]')
expect(jsonrepair('["abc]\n')).toBe('["abc"]\n')
expect(jsonrepair('["abc ]\n')).toBe('["abc" ]\n')
expect(jsonrepair('[\n[\n"abc\n]\n]\n')).toBe('[\n[\n"abc"\n]\n]\n')
})
test('should replace single quotes with double quotes', () => {
expect(jsonrepair("{'a':2}")).toBe('{"a":2}')
expect(jsonrepair("{'a':'foo'}")).toBe('{"a":"foo"}')
expect(jsonrepair('{"a":\'foo\'}')).toBe('{"a":"foo"}')
expect(jsonrepair("{a:'foo',b:'bar'}")).toBe('{"a":"foo","b":"bar"}')
})
test('should replace special quotes with double quotes', () => {
expect(jsonrepair('{“a”:“b”}')).toBe('{"a":"b"}')
expect(jsonrepair('{‘a’:‘b’}')).toBe('{"a":"b"}')
expect(jsonrepair('{`a´:`b´}')).toBe('{"a":"b"}')
})
test('should not replace special quotes inside a normal string', () => {
expect(jsonrepair('"Rounded “ quote"')).toBe('"Rounded “ quote"')
expect(jsonrepair("'Rounded “ quote'")).toBe('"Rounded “ quote"')
expect(jsonrepair('"Rounded ’ quote"')).toBe('"Rounded ’ quote"')
expect(jsonrepair("'Rounded ’ quote'")).toBe('"Rounded ’ quote"')
expect(jsonrepair("'Double \" quote'")).toBe('"Double \\" quote"')
})
test('should not crash when repairing quotes', () => {
expect(jsonrepair("{pattern: '’'}")).toBe('{"pattern": "’"}')
})
test('should leave string content untouched', () => {
expect(jsonrepair('"{a:b}"')).toBe('"{a:b}"')
})
test('should add/remove escape characters', () => {
expect(jsonrepair('"foo\'bar"')).toBe('"foo\'bar"')
expect(jsonrepair('"foo\\"bar"')).toBe('"foo\\"bar"')
expect(jsonrepair("'foo\"bar'")).toBe('"foo\\"bar"')
expect(jsonrepair("'foo\\'bar'")).toBe('"foo\'bar"')
expect(jsonrepair('"foo\\\'bar"')).toBe('"foo\'bar"')
expect(jsonrepair('"\\a"')).toBe('"a"')
})
test('should replace backslash-escaped newline characters', () => {
expect(jsonrepair('"first\\\nsecond"')).toBe('"first\\nsecond"')
})
test('should repair a missing object value', () => {
expect(jsonrepair('{"a":}')).toBe('{"a":null}')
expect(jsonrepair('{"a":,"b":2}')).toBe('{"a":null,"b":2}')
expect(jsonrepair('{"a":')).toBe('{"a":null}')
})
test('should repair undefined values', () => {
expect(jsonrepair('{"a":undefined}')).toBe('{"a":null}')
expect(jsonrepair('[undefined]')).toBe('[null]')
expect(jsonrepair('undefined')).toBe('null')
})
test('should escape unescaped control characters', () => {
expect(jsonrepair('"hello\bworld"')).toBe('"hello\\bworld"')
expect(jsonrepair('"hello\fworld"')).toBe('"hello\\fworld"')
expect(jsonrepair('"hello\nworld"')).toBe('"hello\\nworld"')
expect(jsonrepair('"hello\rworld"')).toBe('"hello\\rworld"')
expect(jsonrepair('"hello\tworld"')).toBe('"hello\\tworld"')
expect(jsonrepair('{"key\nafter": "foo"}')).toBe('{"key\\nafter": "foo"}')
expect(jsonrepair('["hello\nworld"]')).toBe('["hello\\nworld"]')
expect(jsonrepair('["hello\nworld" ]')).toBe('["hello\\nworld" ]')
expect(jsonrepair('["hello\nworld"\n]')).toBe('["hello\\nworld"\n]')
})
test('should escape unescaped double quotes', () => {
expect(jsonrepair('"The TV has a 24" screen"')).toBe('"The TV has a 24\\" screen"')
expect(jsonrepair('{"key": "apple "bee" carrot"}')).toBe('{"key": "apple \\"bee\\" carrot"}')
expect(jsonrepair('[",",":"]')).toBe('[",",":"]')
expect(jsonrepair('["a" 2]')).toBe('["a", 2]')
expect(jsonrepair('["a" 2')).toBe('["a", 2]')
expect(jsonrepair('["," 2')).toBe('[",", 2]')
})
test('should replace special white space characters', () => {
expect(jsonrepair('{"a":\u00a0"foo\u00a0bar"}')).toBe('{"a": "foo\u00a0bar"}')
expect(jsonrepair('{"a":\u180e"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\u2000"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\u2002"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\u200B"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\u202F"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\u205F"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\u3000"foo"}')).toBe('{"a": "foo"}')
expect(jsonrepair('{"a":\ufeff"foo"}')).toBe('{"a": "foo"}')
})
test('should replace non normalized left/right quotes', () => {
expect(jsonrepair('\u2018foo\u2019')).toBe('"foo"')
expect(jsonrepair('\u201Cfoo\u201D')).toBe('"foo"')
expect(jsonrepair('\u0060foo\u00B4')).toBe('"foo"')
// mix single quotes
expect(jsonrepair("\u0060foo'")).toBe('"foo"')
expect(jsonrepair("\u0060foo'")).toBe('"foo"')
})
test('should remove block comments', () => {
expect(jsonrepair('/* foo */ {}')).toBe(' {}')
expect(jsonrepair('{} /* foo */ ')).toBe('{} ')
expect(jsonrepair('{} /* foo ')).toBe('{} ')
expect(jsonrepair('\n/* foo */\n{}')).toBe('\n\n{}')
expect(jsonrepair('{"a":"foo",/*hello*/"b":"bar"}')).toBe('{"a":"foo","b":"bar"}')
expect(jsonrepair('{"flag":/*boolean*/true}')).toBe('{"flag":true}')
})
test('should remove line comments', () => {
expect(jsonrepair('{} // comment')).toBe('{} ')
expect(jsonrepair('{\n"a":"foo",//hello\n"b":"bar"\n}')).toBe('{\n"a":"foo",\n"b":"bar"\n}')
})
test('should not remove comments inside a string', () => {
expect(jsonrepair('"/* foo */"')).toBe('"/* foo */"')
})
test('should remove comments after a string containing a delimiter', () => {
expect(jsonrepair('["a"/* foo */]')).toBe('["a"]')
expect(jsonrepair('["(a)"/* foo */]')).toBe('["(a)"]')
expect(jsonrepair('["a]"/* foo */]')).toBe('["a]"]')
expect(jsonrepair('{"a":"b"/* foo */}')).toBe('{"a":"b"}')
expect(jsonrepair('{"a":"(b)"/* foo */}')).toBe('{"a":"(b)"}')
})
test('should strip JSONP notation', () => {
// matching
expect(jsonrepair('callback_123({});')).toBe('{}')
expect(jsonrepair('callback_123([]);')).toBe('[]')
expect(jsonrepair('callback_123(2);')).toBe('2')
expect(jsonrepair('callback_123("foo");')).toBe('"foo"')
expect(jsonrepair('callback_123(null);')).toBe('null')
expect(jsonrepair('callback_123(true);')).toBe('true')
expect(jsonrepair('callback_123(false);')).toBe('false')
expect(jsonrepair('callback({}')).toBe('{}')
expect(jsonrepair('/* foo bar */ callback_123 ({})')).toBe(' {}')
expect(jsonrepair('/* foo bar */ callback_123 ({})')).toBe(' {}')
expect(jsonrepair('/* foo bar */\ncallback_123({})')).toBe('\n{}')
expect(jsonrepair('/* foo bar */ callback_123 ( {} )')).toBe(' {} ')
expect(jsonrepair(' /* foo bar */ callback_123({}); ')).toBe(' {} ')
expect(jsonrepair('\n/* foo\nbar */\ncallback_123 ({});\n\n')).toBe('\n\n{}\n\n')
// non-matching
expect(() => console.log({ output: jsonrepair('callback {}') })).toThrow(
new JSONRepairError('Unexpected character "{"', 9)
)
})
test('should strip markdown fenced code blocks', () => {
expect(jsonrepair('\`\`\`\n{"a":"b"}\n\`\`\`')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('\`\`\`json\n{"a":"b"}\n\`\`\`')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('\`\`\`\n{"a":"b"}\n')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('\n{"a":"b"}\n\`\`\`')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('\`\`\`{"a":"b"}\`\`\`')).toBe('{"a":"b"}')
expect(jsonrepair('\`\`\`\n[1,2,3]\n\`\`\`')).toBe('\n[1,2,3]\n')
expect(jsonrepair('\`\`\`python\n{"a":"b"}\n\`\`\`')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('\n \`\`\`json\n{"a":"b"}\n\`\`\`\n ')).toBe('\n \n{"a":"b"}\n\n ')
})
test('should strip invalid markdown fenced code blocks', () => {
expect(jsonrepair('[\`\`\`\n{"a":"b"}\n\`\`\`]')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('[\`\`\`json\n{"a":"b"}\n\`\`\`]')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('{\`\`\`\n{"a":"b"}\n\`\`\`}')).toBe('\n{"a":"b"}\n')
expect(jsonrepair('{\`\`\`json\n{"a":"b"}\n\`\`\`}')).toBe('\n{"a":"b"}\n')
})
test('should repair escaped string contents', () => {
expect(jsonrepair('\\"hello world\\"')).toBe('"hello world"')
expect(jsonrepair('\\"hello world\\')).toBe('"hello world"')
expect(jsonrepair('\\"hello \\\\"world\\\\"\\"')).toBe('"hello \\"world\\""')
expect(jsonrepair('[\\"hello \\\\"world\\\\"\\"]')).toBe('["hello \\"world\\""]')
expect(jsonrepair('{\\"stringified\\": \\"hello \\\\"world\\\\"\\"}')).toBe(
'{"stringified": "hello \\"world\\""}'
)
// the following is a bit weird but comes close to the most likely intention
expect(jsonrepair('[\\"hello\\, \\"world\\"]'), '["hello").toBe("world"]')
// the following is sort of invalid: the end quote should be escaped too,
// but the fixed result is most likely what you want in the end
expect(jsonrepair('\\"hello"')).toBe('"hello"')
})
test('should strip a leading comma from an array', () => {
expect(jsonrepair('[,1,2,3]')).toBe('[1,2,3]')
expect(jsonrepair('[/* a */,/* b */1,2,3]')).toBe('[1,2,3]')
expect(jsonrepair('[, 1,2,3]')).toBe('[ 1,2,3]')
expect(jsonrepair('[ , 1,2,3]')).toBe('[ 1,2,3]')
})
test('should strip a leading comma from an object', () => {
expect(jsonrepair('{,"message": "hi"}')).toBe('{"message": "hi"}')
expect(jsonrepair('{/* a */,/* b */"message": "hi"}')).toBe('{"message": "hi"}')
expect(jsonrepair('{ ,"message": "hi"}')).toBe('{ "message": "hi"}')
expect(jsonrepair('{, "message": "hi"}')).toBe('{ "message": "hi"}')
})
test('should strip trailing commas from an array', () => {
expect(jsonrepair('[1,2,3,]')).toBe('[1,2,3]')
expect(jsonrepair('[1,2,3,\n]')).toBe('[1,2,3\n]')
expect(jsonrepair('[1,2,3, \n ]')).toBe('[1,2,3 \n ]')
expect(jsonrepair('[1,2,3,/*foo*/]')).toBe('[1,2,3]')
expect(jsonrepair('{"array":[1,2,3,]}')).toBe('{"array":[1,2,3]}')
// not matching: inside a string
expect(jsonrepair('"[1,2,3,]"')).toBe('"[1,2,3,]"')
})
test('should strip trailing commas from an object', () => {
expect(jsonrepair('{"a":2,}')).toBe('{"a":2}')
expect(jsonrepair('{"a":2 , }')).toBe('{"a":2 }')
expect(jsonrepair('{"a":2 , \n }')).toBe('{"a":2 \n }')
expect(jsonrepair('{"a":2/*foo*/,/*foo*/}')).toBe('{"a":2}')
expect(jsonrepair('{},')).toBe('{}')
// not matching: inside a string
expect(jsonrepair('"{a:2,}"')).toBe('"{a:2,}"')
})
test('should strip trailing comma at the end', () => {
expect(jsonrepair('4,')).toBe('4')
expect(jsonrepair('4 ,')).toBe('4 ')
expect(jsonrepair('4 , ')).toBe('4 ')
expect(jsonrepair('{"a":2},')).toBe('{"a":2}')
expect(jsonrepair('[1,2,3],')).toBe('[1,2,3]')
})
test('should add a missing closing brace for an object', () => {
expect(jsonrepair('{')).toBe('{}')
expect(jsonrepair('{"a":2')).toBe('{"a":2}')
expect(jsonrepair('{"a":2,')).toBe('{"a":2}')
expect(jsonrepair('{"a":{"b":2}')).toBe('{"a":{"b":2}}')
expect(jsonrepair('{\n "a":{"b":2\n}')).toBe('{\n "a":{"b":2\n}}')
expect(jsonrepair('[{"b":2]')).toBe('[{"b":2}]')
expect(jsonrepair('[{"b":2\n]')).toBe('[{"b":2}\n]')
expect(jsonrepair('[{"i":1{"i":2}]')).toBe('[{"i":1},{"i":2}]')
expect(jsonrepair('[{"i":1,{"i":2}]')).toBe('[{"i":1},{"i":2}]')
})
test('should remove a redundant closing bracket for an object', () => {
expect(jsonrepair('{"a": 1}}')).toBe('{"a": 1}')
expect(jsonrepair('{"a": 1}}]}')).toBe('{"a": 1}')
expect(jsonrepair('{"a": 1 } } ] } ')).toBe('{"a": 1 } ')
expect(jsonrepair('{"a":2]')).toBe('{"a":2}')
expect(jsonrepair('{"a":2,]')).toBe('{"a":2}')
expect(jsonrepair('{}}')).toBe('{}')
expect(jsonrepair('[2,}')).toBe('[2]')
expect(jsonrepair('[}')).toBe('[]')
expect(jsonrepair('{]')).toBe('{}')
})
test('should add a missing closing bracket for an array', () => {
expect(jsonrepair('[')).toBe('[]')
expect(jsonrepair('[1,2,3')).toBe('[1,2,3]')
expect(jsonrepair('[1,2,3,')).toBe('[1,2,3]')
expect(jsonrepair('[[1,2,3,')).toBe('[[1,2,3]]')
expect(jsonrepair('{\n"values":[1,2,3\n}')).toBe('{\n"values":[1,2,3]\n}')
expect(jsonrepair('{\n"values":[1,2,3\n')).toBe('{\n"values":[1,2,3]}\n')
})
test('should strip MongoDB data types', () => {
// simple
expect(jsonrepair('NumberLong("2")')).toBe('"2"')
expect(jsonrepair('{"_id":ObjectId("123")}')).toBe('{"_id":"123"}')
// extensive
const mongoDocument =
'{\n' +
' "_id" : ObjectId("123"),\n' +
' "isoDate" : ISODate("2012-12-19T06:01:17.171Z"),\n' +
' "regularNumber" : 67,\n' +
' "long" : NumberLong("2"),\n' +
' "long2" : NumberLong(2),\n' +
' "int" : NumberInt("3"),\n' +
' "int2" : NumberInt(3),\n' +
' "decimal" : NumberDecimal("4"),\n' +
' "decimal2" : NumberDecimal(4)\n' +
'}'
const expectedJson =
'{\n' +
' "_id" : "123",\n' +
' "isoDate" : "2012-12-19T06:01:17.171Z",\n' +
' "regularNumber" : 67,\n' +
' "long" : "2",\n' +
' "long2" : 2,\n' +
' "int" : "3",\n' +
' "int2" : 3,\n' +
' "decimal" : "4",\n' +
' "decimal2" : 4\n' +
'}'
expect(jsonrepair(mongoDocument)).toBe(expectedJson)
})
test('should parse an unquoted string', () => {
expect(jsonrepair('hello world')).toBe('"hello world"')
expect(jsonrepair('She said: no way')).toBe('"She said: no way"')
expect(jsonrepair('["This is C(2)", "This is F(3)]')).toBe('["This is C(2)", "This is F(3)"]')
expect(jsonrepair('["This is C(2)", This is F(3)]')).toBe('["This is C(2)", "This is F(3)"]')
})
test('should replace Python constants None, True, False', () => {
expect(jsonrepair('True')).toBe('true')
expect(jsonrepair('False')).toBe('false')
expect(jsonrepair('None')).toBe('null')
})
test('should turn unknown symbols into a string', () => {
expect(jsonrepair('foo')).toBe('"foo"')
expect(jsonrepair('[1,foo,4]')).toBe('[1,"foo",4]')
expect(jsonrepair('{foo: bar}')).toBe('{"foo": "bar"}')
expect(jsonrepair('foo 2 bar')).toBe('"foo 2 bar"')
expect(jsonrepair('{greeting: hello world}')).toBe('{"greeting": "hello world"}')
expect(jsonrepair('{greeting: hello world\nnext: "line"}')).toBe(
'{"greeting": "hello world",\n"next": "line"}'
)
expect(jsonrepair('{greeting: hello world!}')).toBe('{"greeting": "hello world!"}')
})
test('should turn invalid numbers into strings', () => {
expect(jsonrepair('ES2020')).toBe('"ES2020"')
expect(jsonrepair('0.0.1')).toBe('"0.0.1"')
expect(jsonrepair('746de9ad-d4ff-4c66-97d7-00a92ad46967')).toBe(
'"746de9ad-d4ff-4c66-97d7-00a92ad46967"'
)
expect(jsonrepair('234..5')).toBe('"234..5"')
expect(jsonrepair('[0.0.1,2]')).toBe('["0.0.1",2]') // test delimiter for numerics
expect(jsonrepair('[2 0.0.1 2]')).toBe('[2, "0.0.1 2"]') // note: currently spaces delimit numbers, but don't delimit unquoted strings
expect(jsonrepair('2e3.4')).toBe('"2e3.4"')
})
test('should repair regular expressions', () => {
expect(jsonrepair('{regex: /standalone-styles.css/}')).toBe(
'{"regex": "/standalone-styles.css/"}'
)
expect(jsonrepair('/[a-z]_/')).toBe('"/[a-z]_/"')
// with escape char
const repairedRegex = jsonrepair('/\\//')
expect(repairedRegex).toEqual('"/\\\\//"')
const parsedRegex = JSON.parse(repairedRegex)
const regex = new RegExp(parsedRegex.substring(1, parsedRegex.length - 1)) // remove the outer slashes /.../ with substring
expect(regex.test('/')).toEqual(true)
})
test('should escape quotes in repaired regular expressions', () => {
// Prevent a string like:
// '/foo"; console.log(-1); "/'
// from being parsed into:
// '"/foo"; console.log(-1); "/"'
// which would be executed as JavaScript when this JSON is being parsed with `eval`.
// See https://github.com/josdejong/jsonrepair/issues/150
// Note: using `eval` introduces security risks in general, best is to avoid it.
expect(jsonrepair('/foo"; console.log(-1); "/')).toBe('"/foo\\"; console.log(-1); \\"/"')
})
test('should concatenate strings', () => {
expect(jsonrepair('"hello" + " world"')).toBe('"hello world"')
expect(jsonrepair('"hello" +\n " world"')).toBe('"hello world"')
expect(jsonrepair('"a"+"b"+"c"')).toBe('"abc"')
expect(jsonrepair('"hello" + /*comment*/ " world"')).toBe('"hello world"')
expect(jsonrepair("{\n \"greeting\": 'hello' +\n 'world'\n}")).toBe(
'{\n "greeting": "helloworld"\n}'
)
expect(jsonrepair('"hello +\n " world"')).toBe('"hello world"')
expect(jsonrepair('"hello +')).toBe('"hello"')
expect(jsonrepair('["hello +]')).toBe('["hello"]')
})
test('should repair missing comma between array items', () => {
expect(jsonrepair('{"array": [{}{}]}')).toBe('{"array": [{},{}]}')
expect(jsonrepair('{"array": [{} {}]}'), '{"array": [{}).toBe({}]}')
expect(jsonrepair('{"array": [{}\n{}]}')).toBe('{"array": [{},\n{}]}')
expect(jsonrepair('{"array": [\n{}\n{}\n]}')).toBe('{"array": [\n{},\n{}\n]}')
expect(jsonrepair('{"array": [\n1\n2\n]}')).toBe('{"array": [\n1,\n2\n]}')
expect(jsonrepair('{"array": [\n"a"\n"b"\n]}')).toBe('{"array": [\n"a",\n"b"\n]}')
// should leave normal array as is
expect(jsonrepair('[\n{},\n{}\n]')).toBe('[\n{},\n{}\n]')
})
test('should repair missing comma between object properties', () => {
expect(jsonrepair('{"a":2\n"b":3\n}')).toBe('{"a":2,\n"b":3\n}')
expect(jsonrepair('{"a":2\n"b":3\nc:4}')).toBe('{"a":2,\n"b":3,\n"c":4}')
expect(jsonrepair('{\n "firstName": "John"\n lastName: Smith')).toBe(
'{\n "firstName": "John",\n "lastName": "Smith"}'
)
expect(jsonrepair('{\n "firstName": "John" /* comment */ \n lastName: Smith')).toBe(
'{\n "firstName": "John", \n "lastName": "Smith"}'
)
// verify parsing a comma after a return (since in parseString we stop at a return)
expect(jsonrepair('{\n "firstName": "John"\n , lastName: Smith')).toBe(
'{\n "firstName": "John"\n , "lastName": "Smith"}'
)
})
test('should repair numbers at the end', () => {
expect(jsonrepair('{"a":2.')).toBe('{"a":2.0}')
expect(jsonrepair('{"a":2e')).toBe('{"a":2e0}')
expect(jsonrepair('{"a":2e-')).toBe('{"a":2e-0}')
expect(jsonrepair('{"a":-')).toBe('{"a":-0}')
expect(jsonrepair('[2e,')).toBe('[2e0]')
expect(jsonrepair('[2e ')).toBe('[2e0] ') // spaces delimit numbers
expect(jsonrepair('[-,')).toBe('[-0]')
})
test('should repair missing colon between object key and value', () => {
expect(jsonrepair('{"a" "b"}')).toBe('{"a": "b"}')
expect(jsonrepair('{"a" 2}')).toBe('{"a": 2}')
expect(jsonrepair('{"a" true}')).toBe('{"a": true}')
expect(jsonrepair('{"a" false}')).toBe('{"a": false}')
expect(jsonrepair('{"a" null}')).toBe('{"a": null}')
expect(jsonrepair('{"a"2}')).toBe('{"a":2}')
expect(jsonrepair('{\n"a" "b"\n}')).toBe('{\n"a": "b"\n}')
expect(jsonrepair('{"a" \'b\'}')).toBe('{"a": "b"}')
expect(jsonrepair("{'a' 'b'}")).toBe('{"a": "b"}')
expect(jsonrepair('{“a” “b”}')).toBe('{"a": "b"}')
expect(jsonrepair("{a 'b'}")).toBe('{"a": "b"}')
expect(jsonrepair('{a “b”}')).toBe('{"a": "b"}')
})
test('should repair missing a combination of comma, quotes and brackets', () => {
expect(jsonrepair('{"array": [\na\nb\n]}')).toBe('{"array": [\n"a",\n"b"\n]}')
expect(jsonrepair('1\n2')).toBe('[\n1,\n2\n]')
expect(jsonrepair('[a,b\nc]')).toBe('["a","b",\n"c"]')
})
test('should repair newline separated json (for example from MongoDB)', () => {
const text =
'' + '/* 1 */\n' + '{}\n' + '\n' + '/* 2 */\n' + '{}\n' + '\n' + '/* 3 */\n' + '{}\n'
const expected = '[\n\n{},\n\n\n{},\n\n\n{}\n\n]'
expect(jsonrepair(text)).toBe(expected)
})
test('should repair newline separated json having commas', () => {
const text =
'' + '/* 1 */\n' + '{},\n' + '\n' + '/* 2 */\n' + '{},\n' + '\n' + '/* 3 */\n' + '{}\n'
const expected = '[\n\n{},\n\n\n{},\n\n\n{}\n\n]'
expect(jsonrepair(text)).toBe(expected)
})
test('should repair newline separated json having commas and trailing comma', () => {
const text =
'' + '/* 1 */\n' + '{},\n' + '\n' + '/* 2 */\n' + '{},\n' + '\n' + '/* 3 */\n' + '{},\n'
const expected = '[\n\n{},\n\n\n{},\n\n\n{}\n\n]'
expect(jsonrepair(text)).toBe(expected)
})
test('should repair a comma separated list with value', () => {
expect(jsonrepair('1,2,3')).toBe('[\n1,2,3\n]')
expect(jsonrepair('1,2,3,')).toBe('[\n1,2,3\n]')
expect(jsonrepair('1\n2\n3')).toBe('[\n1,\n2,\n3\n]')
expect(jsonrepair('a\nb')).toBe('[\n"a",\n"b"\n]')
expect(jsonrepair('a,b')).toBe('[\n"a","b"\n]')
})
test('should repair a number with leading zero', () => {
expect(jsonrepair('0789')).toBe('"0789"')
expect(jsonrepair('000789')).toBe('"000789"')
expect(jsonrepair('001.2')).toBe('"001.2"')
expect(jsonrepair('002e3')).toBe('"002e3"')
expect(jsonrepair('[0789]')).toBe('["0789"]')
expect(jsonrepair('{value:0789}')).toBe('{"value":"0789"}')
})
})
test('should throw an exception in case of non-repairable issues', () => {
expect(() => {
console.log({ output: jsonrepair('') })
}).toThrow(new JSONRepairError('Unexpected end of json string', 0))
expect(() => {
console.log({ output: jsonrepair('{"a",') })
}).toThrow(new JSONRepairError('Colon expected', 4))
expect(() => {
console.log({ output: jsonrepair('{:2}') })
}).toThrow(new JSONRepairError('Object key expected', 1))
expect(() => {
console.log({ output: jsonrepair('{"a":2}{}') })
}).toThrow(new JSONRepairError('Unexpected character "{"', 7))
expect(() => {
console.log({ output: jsonrepair('{"a" ]') })
}).toThrow(new JSONRepairError('Colon expected', 5))
expect(() => {
console.log({ output: jsonrepair('{"a":2}foo') })
}).toThrow(new JSONRepairError('Unexpected character "f"', 7))
expect(() => {
console.log({ output: jsonrepair('foo [') })
}).toThrow(new JSONRepairError('Unexpected character "["', 4))
expect(() => {
console.log({ output: jsonrepair('"\\u26"') })
}).toThrow(new JSONRepairError('Invalid unicode character "\\u26""', 1))
expect(() => {
console.log({ output: jsonrepair('"\\uZ000"') })
}).toThrow(new JSONRepairError('Invalid unicode character "\\uZ000"', 1))
expect(() => {
console.log({ output: jsonrepair('"\\uZ000') })
}).toThrow(new JSONRepairError('Invalid unicode character "\\uZ000"', 1))
expect(() => {
console.log({ output: jsonrepair('"abc\u{00}"') })
}).toThrow(new JSONRepairError('Invalid character "\\u0000"', 4))
expect(() => {
console.log({ output: jsonrepair('"abc\u{1f}"') })
}).toThrow(new JSONRepairError('Invalid character "\\u001f"', 4))
})
function assertRepair(text: string) {
expect(jsonrepair(text)).toEqual(text)
}
})
function createStreamingRepairWrapper(): (input: string) => string {
return function jsonrepair(text: string): string {
let output = ''
// Note: without an infinite bufferSize and chunkSize, the function
// is faster, but it can potentially through an "Index out of range"
// error, and we do not want that.
const { transform, flush } = jsonrepairCore({
onData: (chunk) => {
output += chunk
},
bufferSize: Number.POSITIVE_INFINITY,
chunkSize: Number.POSITIVE_INFINITY
})
transform(text)
flush()
return output
}
}
```
## /src/index.ts
```ts path="/src/index.ts"
// Cross-platform, non-streaming JavaScript API
export { jsonrepair } from './regular/jsonrepair.js'
export { JSONRepairError } from './utils/JSONRepairError.js'
```
## /src/regular/jsonrepair.ts
```ts path="/src/regular/jsonrepair.ts"
import { JSONRepairError } from '../utils/JSONRepairError.js'
import {
endsWithCommaOrNewline,
insertBeforeLastWhitespace,
isControlCharacter,
isDelimiter,
isDigit,
isDoubleQuote,
isDoubleQuoteLike,
isFunctionNameChar,
isFunctionNameCharStart,
isHex,
isQuote,
isSingleQuote,
isSingleQuoteLike,
isSpecialWhitespace,
isStartOfValue,
isUnquotedStringDelimiter,
isValidStringCharacter,
isWhitespace,
isWhitespaceExceptNewline,
regexUrlChar,
regexUrlStart,
removeAtIndex,
stripLastOccurrence
} from '../utils/stringUtils.js'
const controlCharacters: { [key: string]: string } = {
'\b': '\\b',
'\f': '\\f',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t'
}
// map with all escape characters
const escapeCharacters: { [key: string]: string } = {
'"': '"',
'\\': '\\',
'/': '/',
b: '\b',
f: '\f',
n: '\n',
r: '\r',
t: '\t'
// note that \u is handled separately in parseString()
}
/**
* Repair a string containing an invalid JSON document.
* For example changes JavaScript notation into JSON notation.
*
* Example:
*
* try {
* const json = "{name: 'John'}"
* const repaired = jsonrepair(json)
* console.log(repaired)
* // '{"name": "John"}'
* } catch (err) {
* console.error(err)
* }
*
*/
export function jsonrepair(text: string): string {
let i = 0 // current index in text
let output = '' // generated output
parseMarkdownCodeBlock(['\`\`\`', '[\`\`\`', '{\`\`\`'])
const processed = parseValue()
if (!processed) {
throwUnexpectedEnd()
}
parseMarkdownCodeBlock(['\`\`\`', '\`\`\`]', '\`\`\`}'])
const processedComma = parseCharacter(',')
if (processedComma) {
parseWhitespaceAndSkipComments()
}
if (isStartOfValue(text[i]) && endsWithCommaOrNewline(output)) {
// start of a new value after end of the root level object: looks like
// newline delimited JSON -> turn into a root level array
if (!processedComma) {
// repair missing comma
output = insertBeforeLastWhitespace(output, ',')
}
parseNewlineDelimitedJSON()
} else if (processedComma) {
// repair: remove trailing comma
output = stripLastOccurrence(output, ',')
}
// repair redundant end quotes
while (text[i] === '}' || text[i] === ']') {
i++
parseWhitespaceAndSkipComments()
}
if (i >= text.length) {
// reached the end of the document properly
return output
}
throwUnexpectedCharacter()
function parseValue(): boolean {
parseWhitespaceAndSkipComments()
const processed =
parseObject() ||
parseArray() ||
parseString() ||
parseNumber() ||
parseKeywords() ||
parseUnquotedString(false) ||
parseRegex()
parseWhitespaceAndSkipComments()
return processed
}
function parseWhitespaceAndSkipComments(skipNewline = true): boolean {
const start = i
let changed = parseWhitespace(skipNewline)
do {
changed = parseComment()
if (changed) {
changed = parseWhitespace(skipNewline)
}
} while (changed)
return i > start
}
function parseWhitespace(skipNewline: boolean): boolean {
const _isWhiteSpace = skipNewline ? isWhitespace : isWhitespaceExceptNewline
let whitespace = ''
while (true) {
if (_isWhiteSpace(text, i)) {
whitespace += text[i]
i++
} else if (isSpecialWhitespace(text, i)) {
// repair special whitespace
whitespace += ' '
i++
} else {
break
}
}
if (whitespace.length > 0) {
output += whitespace
return true
}
return false
}
function parseComment(): boolean {
// find a block comment '/* ... */'
if (text[i] === '/' && text[i + 1] === '*') {
// repair block comment by skipping it
while (i < text.length && !atEndOfBlockComment(text, i)) {
i++
}
i += 2
return true
}
// find a line comment '// ...'
if (text[i] === '/' && text[i + 1] === '/') {
// repair line comment by skipping it
while (i < text.length && text[i] !== '\n') {
i++
}
return true
}
return false
}
function parseMarkdownCodeBlock(blocks: string[]): boolean {
// find and skip over a Markdown fenced code block:
// \`\`\` ... \`\`\`
// or
// \`\`\`json ... \`\`\`
if (skipMarkdownCodeBlock(blocks)) {
if (isFunctionNameCharStart(text[i])) {
// strip the optional language specifier like "json"
while (i < text.length && isFunctionNameChar(text[i])) {
i++
}
}
parseWhitespaceAndSkipComments()
return true
}
return false
}
function skipMarkdownCodeBlock(blocks: string[]): boolean {
parseWhitespace(true)
for (const block of blocks) {
const end = i + block.length
if (text.slice(i, end) === block) {
i = end
return true
}
}
return false
}
function parseCharacter(char: string): boolean {
if (text[i] === char) {
output += text[i]
i++
return true
}
return false
}
function skipCharacter(char: string): boolean {
if (text[i] === char) {
i++
return true
}
return false
}
function skipEscapeCharacter(): boolean {
return skipCharacter('\\')
}
/**
* Skip ellipsis like "[1,2,3,...]" or "[1,2,3,...,9]" or "[...,7,8,9]"
* or a similar construct in objects.
*/
function skipEllipsis(): boolean {
parseWhitespaceAndSkipComments()
if (text[i] === '.' && text[i + 1] === '.' && text[i + 2] === '.') {
// repair: remove the ellipsis (three dots) and optionally a comma
i += 3
parseWhitespaceAndSkipComments()
skipCharacter(',')
return true
}
return false
}
/**
* Parse an object like '{"key": "value"}'
*/
function parseObject(): boolean {
if (text[i] === '{') {
output += '{'
i++
parseWhitespaceAndSkipComments()
// repair: skip leading comma like in {, message: "hi"}
if (skipCharacter(',')) {
parseWhitespaceAndSkipComments()
}
let initial = true
while (i < text.length && text[i] !== '}') {
let processedComma: boolean
if (!initial) {
processedComma = parseCharacter(',')
if (!processedComma) {
// repair missing comma
output = insertBeforeLastWhitespace(output, ',')
}
parseWhitespaceAndSkipComments()
} else {
processedComma = true
initial = false
}
skipEllipsis()
const processedKey = parseString() || parseUnquotedString(true)
if (!processedKey) {
if (
text[i] === '}' ||
text[i] === '{' ||
text[i] === ']' ||
text[i] === '[' ||
text[i] === undefined
) {
// repair trailing comma
output = stripLastOccurrence(output, ',')
} else {
throwObjectKeyExpected()
}
break
}
parseWhitespaceAndSkipComments()
const processedColon = parseCharacter(':')
const truncatedText = i >= text.length
if (!processedColon) {
if (isStartOfValue(text[i]) || truncatedText) {
// repair missing colon
output = insertBeforeLastWhitespace(output, ':')
} else {
throwColonExpected()
}
}
const processedValue = parseValue()
if (!processedValue) {
if (processedColon || truncatedText) {
// repair missing object value
output += 'null'
} else {
throwColonExpected()
}
}
}
if (text[i] === '}') {
output += '}'
i++
} else {
// repair missing end bracket
output = insertBeforeLastWhitespace(output, '}')
}
return true
}
return false
}
/**
* Parse an array like '["item1", "item2", ...]'
*/
function parseArray(): boolean {
if (text[i] === '[') {
output += '['
i++
parseWhitespaceAndSkipComments()
// repair: skip leading comma like in [,1,2,3]
if (skipCharacter(',')) {
parseWhitespaceAndSkipComments()
}
let initial = true
while (i < text.length && text[i] !== ']') {
if (!initial) {
const processedComma = parseCharacter(',')
if (!processedComma) {
// repair missing comma
output = insertBeforeLastWhitespace(output, ',')
}
} else {
initial = false
}
skipEllipsis()
const processedValue = parseValue()
if (!processedValue) {
// repair trailing comma
output = stripLastOccurrence(output, ',')
break
}
}
if (text[i] === ']') {
output += ']'
i++
} else {
// repair missing closing array bracket
output = insertBeforeLastWhitespace(output, ']')
}
return true
}
return false
}
/**
* Parse and repair Newline Delimited JSON (NDJSON):
* multiple JSON objects separated by a newline character
*/
function parseNewlineDelimitedJSON() {
// repair NDJSON
let initial = true
let processedValue = true
while (processedValue) {
if (!initial) {
// parse optional comma, insert when missing
const processedComma = parseCharacter(',')
if (!processedComma) {
// repair: add missing comma
output = insertBeforeLastWhitespace(output, ',')
}
} else {
initial = false
}
processedValue = parseValue()
}
if (!processedValue) {
// repair: remove trailing comma
output = stripLastOccurrence(output, ',')
}
// repair: wrap the output inside array brackets
output = `[\n${output}\n]`
}
/**
* Parse a string enclosed by double quotes "...". Can contain escaped quotes
* Repair strings enclosed in single quotes or special quotes
* Repair an escaped string
*
* The function can run in two stages:
* - First, it assumes the string has a valid end quote
* - If it turns out that the string does not have a valid end quote followed
* by a delimiter (which should be the case), the function runs again in a
* more conservative way, stopping the string at the first next delimiter
* and fixing the string by inserting a quote there, or stopping at a
* stop index detected in the first iteration.
*/
function parseString(stopAtDelimiter = false, stopAtIndex = -1): boolean {
let skipEscapeChars = text[i] === '\\'
if (skipEscapeChars) {
// repair: remove the first escape character
i++
skipEscapeChars = true
}
if (isQuote(text[i])) {
// double quotes are correct JSON,
// single quotes come from JavaScript for example, we assume it will have a correct single end quote too
// otherwise, we will match any double-quote-like start with a double-quote-like end,
// or any single-quote-like start with a single-quote-like end
const isEndQuote = isDoubleQuote(text[i])
? isDoubleQuote
: isSingleQuote(text[i])
? isSingleQuote
: isSingleQuoteLike(text[i])
? isSingleQuoteLike
: isDoubleQuoteLike
const iBefore = i
const oBefore = output.length
let str = '"'
i++
while (true) {
if (i >= text.length) {
// end of text, we are missing an end quote
const iPrev = prevNonWhitespaceIndex(i - 1)
if (!stopAtDelimiter && isDelimiter(text.charAt(iPrev))) {
// if the text ends with a delimiter, like ["hello],
// so the missing end quote should be inserted before this delimiter
// retry parsing the string, stopping at the first next delimiter
i = iBefore
output = output.substring(0, oBefore)
return parseString(true)
}
// repair missing quote
str = insertBeforeLastWhitespace(str, '"')
output += str
return true
}
if (i === stopAtIndex) {
// use the stop index detected in the first iteration, and repair end quote
str = insertBeforeLastWhitespace(str, '"')
output += str
return true
}
if (isEndQuote(text[i])) {
// end quote
// let us check what is before and after the quote to verify whether this is a legit end quote
const iQuote = i
const oQuote = str.length
str += '"'
i++
output += str
parseWhitespaceAndSkipComments(false)
if (
stopAtDelimiter ||
i >= text.length ||
isDelimiter(text[i]) ||
isQuote(text[i]) ||
isDigit(text[i])
) {
// The quote is followed by the end of the text, a delimiter,
// or a next value. So the quote is indeed the end of the string.
parseConcatenatedString()
return true
}
const iPrevChar = prevNonWhitespaceIndex(iQuote - 1)
const prevChar = text.charAt(iPrevChar)
if (prevChar === ',') {
// A comma followed by a quote, like '{"a":"b,c,"d":"e"}'.
// We assume that the quote is a start quote, and that the end quote
// should have been located right before the comma but is missing.
i = iBefore
output = output.substring(0, oBefore)
return parseString(false, iPrevChar)
}
if (isDelimiter(prevChar)) {
// This is not the right end quote: it is preceded by a delimiter,
// and NOT followed by a delimiter. So, there is an end quote missing
// parse the string again and then stop at the first next delimiter
i = iBefore
output = output.substring(0, oBefore)
return parseString(true)
}
// revert to right after the quote but before any whitespace, and continue parsing the string
output = output.substring(0, oBefore)
i = iQuote + 1
// repair unescaped quote
str = `${str.substring(0, oQuote)}\\${str.substring(oQuote)}`
} else if (stopAtDelimiter && isUnquotedStringDelimiter(text[i])) {
// we're in the mode to stop the string at the first delimiter
// because there is an end quote missing
// test start of an url like "https://..." (this would be parsed as a comment)
if (text[i - 1] === ':' && regexUrlStart.test(text.substring(iBefore + 1, i + 2))) {
while (i < text.length && regexUrlChar.test(text[i])) {
str += text[i]
i++
}
}
// repair missing quote
str = insertBeforeLastWhitespace(str, '"')
output += str
parseConcatenatedString()
return true
} else if (text[i] === '\\') {
// handle escaped content like \n or \u2605
const char = text.charAt(i + 1)
const escapeChar = escapeCharacters[char]
if (escapeChar !== undefined) {
str += text.slice(i, i + 2)
i += 2
} else if (char === 'u') {
let j = 2
while (j < 6 && isHex(text[i + j])) {
j++
}
if (j === 6) {
str += text.slice(i, i + 6)
i += 6
} else if (i + j >= text.length) {
// repair invalid or truncated unicode char at the end of the text
// by removing the unicode char and ending the string here
i = text.length
} else {
throwInvalidUnicodeCharacter()
}
} else if (char === '\n') {
// repair a backslash escaped newline (like in Bash scripts)
str += '\\n'
i += 2
} else {
// repair invalid escape character: remove it
str += char
i += 2
}
} else {
// handle regular characters
const char = text.charAt(i)
if (char === '"' && text[i - 1] !== '\\') {
// repair unescaped double quote
str += `\\${char}`
i++
} else if (isControlCharacter(char)) {
// unescaped control character
str += controlCharacters[char]
i++
} else {
if (!isValidStringCharacter(char)) {
throwInvalidCharacter(char)
}
str += char
i++
}
}
if (skipEscapeChars) {
// repair: skipped escape character (nothing to do)
skipEscapeCharacter()
}
}
}
return false
}
/**
* Repair concatenated strings like "hello" + "world", change this into "helloworld"
*/
function parseConcatenatedString(): boolean {
let processed = false
parseWhitespaceAndSkipComments()
while (text[i] === '+') {
processed = true
i++
parseWhitespaceAndSkipComments()
// repair: remove the end quote of the first string
output = stripLastOccurrence(output, '"', true)
const start = output.length
const parsedStr = parseString()
if (parsedStr) {
// repair: remove the start quote of the second string
output = removeAtIndex(output, start, 1)
} else {
// repair: remove the + because it is not followed by a string
output = insertBeforeLastWhitespace(output, '"')
}
}
return processed
}
/**
* Parse a number like 2.4 or 2.4e6
*/
function parseNumber(): boolean {
const start = i
if (text[i] === '-') {
i++
if (atEndOfNumber()) {
repairNumberEndingWithNumericSymbol(start)
return true
}
if (!isDigit(text[i])) {
i = start
return false
}
}
// Note that in JSON leading zeros like "00789" are not allowed.
// We will allow all leading zeros here though and at the end of parseNumber
// check against trailing zeros and repair that if needed.
// Leading zeros can have meaning, so we should not clear them.
while (isDigit(text[i])) {
i++
}
if (text[i] === '.') {
i++
if (atEndOfNumber()) {
repairNumberEndingWithNumericSymbol(start)
return true
}
if (!isDigit(text[i])) {
i = start
return false
}
while (isDigit(text[i])) {
i++
}
}
if (text[i] === 'e' || text[i] === 'E') {
i++
if (text[i] === '-' || text[i] === '+') {
i++
}
if (atEndOfNumber()) {
repairNumberEndingWithNumericSymbol(start)
return true
}
if (!isDigit(text[i])) {
i = start
return false
}
while (isDigit(text[i])) {
i++
}
}
// if we're not at the end of the number by this point, allow this to be parsed as another type
if (!atEndOfNumber()) {
i = start
return false
}
if (i > start) {
// repair a number with leading zeros like "00789"
const num = text.slice(start, i)
const hasInvalidLeadingZero = /^0\d/.test(num)
output += hasInvalidLeadingZero ? `"${num}"` : num
return true
}
return false
}
/**
* Parse keywords true, false, null
* Repair Python keywords True, False, None
*/
function parseKeywords(): boolean {
return (
parseKeyword('true', 'true') ||
parseKeyword('false', 'false') ||
parseKeyword('null', 'null') ||
// repair Python keywords True, False, None
parseKeyword('True', 'true') ||
parseKeyword('False', 'false') ||
parseKeyword('None', 'null')
)
}
function parseKeyword(name: string, value: string): boolean {
if (text.slice(i, i + name.length) === name) {
output += value
i += name.length
return true
}
return false
}
/**
* Repair an unquoted string by adding quotes around it
* Repair a MongoDB function call like NumberLong("2")
* Repair a JSONP function call like callback({...});
*/
function parseUnquotedString(isKey: boolean) {
// note that the symbol can end with whitespaces: we stop at the next delimiter
// also, note that we allow strings to contain a slash / in order to support repairing regular expressions
const start = i
if (isFunctionNameCharStart(text[i])) {
while (i < text.length && isFunctionNameChar(text[i])) {
i++
}
let j = i
while (isWhitespace(text, j)) {
j++
}
if (text[j] === '(') {
// repair a MongoDB function call like NumberLong("2")
// repair a JSONP function call like callback({...});
i = j + 1
parseValue()
if (text[i] === ')') {
// repair: skip close bracket of function call
i++
if (text[i] === ';') {
// repair: skip semicolon after JSONP call
i++
}
}
return true
}
}
while (
i < text.length &&
!isUnquotedStringDelimiter(text[i]) &&
!isQuote(text[i]) &&
(!isKey || text[i] !== ':')
) {
i++
}
// test start of an url like "https://..." (this would be parsed as a comment)
if (text[i - 1] === ':' && regexUrlStart.test(text.substring(start, i + 2))) {
while (i < text.length && regexUrlChar.test(text[i])) {
i++
}
}
if (i > start) {
// repair unquoted string
// also, repair undefined into null
// first, go back to prevent getting trailing whitespaces in the string
while (isWhitespace(text, i - 1) && i > 0) {
i--
}
const symbol = text.slice(start, i)
output += symbol === 'undefined' ? 'null' : JSON.stringify(symbol)
if (text[i] === '"') {
// we had a missing start quote, but now we encountered the end quote, so we can skip that one
i++
}
return true
}
}
function parseRegex() {
if (text[i] === '/') {
const start = i
i++
while (i < text.length && (text[i] !== '/' || text[i - 1] === '\\')) {
i++
}
i++
output += JSON.stringify(text.substring(start, i))
return true
}
}
function prevNonWhitespaceIndex(start: number): number {
let prev = start
while (prev > 0 && isWhitespace(text, prev)) {
prev--
}
return prev
}
function atEndOfNumber() {
return i >= text.length || isDelimiter(text[i]) || isWhitespace(text, i)
}
function repairNumberEndingWithNumericSymbol(start: number) {
// repair numbers cut off at the end
// this will only be called when we end after a '.', '-', or 'e' and does not
// change the number more than it needs to make it valid JSON
output += `${text.slice(start, i)}0`
}
function throwInvalidCharacter(char: string) {
throw new JSONRepairError(`Invalid character ${JSON.stringify(char)}`, i)
}
function throwUnexpectedCharacter() {
throw new JSONRepairError(`Unexpected character ${JSON.stringify(text[i])}`, i)
}
function throwUnexpectedEnd() {
throw new JSONRepairError('Unexpected end of json string', text.length)
}
function throwObjectKeyExpected() {
throw new JSONRepairError('Object key expected', i)
}
function throwColonExpected() {
throw new JSONRepairError('Colon expected', i)
}
function throwInvalidUnicodeCharacter() {
const chars = text.slice(i, i + 6)
throw new JSONRepairError(`Invalid unicode character "${chars}"`, i)
}
}
function atEndOfBlockComment(text: string, i: number) {
return text[i] === '*' && text[i + 1] === '/'
}
```
## /src/stream.ts
```ts path="/src/stream.ts"
// Node.js streaming API
export { type JsonRepairTransformOptions, jsonrepairTransform } from './streaming/stream.js'
```
## /src/streaming/buffer/InputBuffer.test.ts
```ts path="/src/streaming/buffer/InputBuffer.test.ts"
import { describe, expect, test } from 'vitest'
import { createInputBuffer } from './InputBuffer'
describe('InputBuffer', () => {
test('should read bytes via charAt', () => {
const { buffer } = testInputBuffer('0123456789')
buffer.close()
expect(buffer.charAt(3)).toBe('3')
expect(buffer.charAt(4)).toBe('4')
expect(buffer.charAt(6)).toBe('6')
expect(buffer.charAt(9)).toBe('9')
expect(buffer.charAt(10)).toBe('')
expect(buffer.charAt(100)).toBe('')
})
test('should throw when using charAt when the index is already flushed', () => {
const { buffer } = testInputBuffer('0123456789')
buffer.flush(2)
expect(() => buffer.charAt(1)).toThrow(
/Index out of range, please configure a larger buffer size \(index: 1, offset: 2\)/
)
expect(buffer.charAt(2)).toBe('2')
})
test('should read bytes via charCodeAt', () => {
const { buffer } = testInputBuffer('0123456789')
buffer.close()
expect(buffer.charCodeAt(3)).toBe('3'.charCodeAt(0))
expect(buffer.charCodeAt(8)).toBe('8'.charCodeAt(0))
expect(buffer.charCodeAt(12)).toBe(Number.NaN)
})
test('should get a substring', () => {
const { buffer } = testInputBuffer('0123456789')
expect(buffer.substring(3, 5)).toBe('34')
})
test('should get a substring with limited buffer size', () => {
const { buffer } = testInputBuffer('0123456789')
expect(buffer.substring(3, 5)).toBe('34')
buffer.flush(5)
expect(() => buffer.substring(0, 1)).toThrow(
/Index out of range, please configure a larger buffer size \(index: 0, offset: 5\)/
)
expect(() => buffer.substring(4, 9)).toThrow(
/Index out of range, please configure a larger buffer size \(index: 4, offset: 5\)/
)
})
test('should get the length', () => {
const { buffer } = testInputBuffer('0123456789')
expect(() => buffer.length()).toThrow(/Cannot get length: input is not yet closed/)
buffer.close()
expect(buffer.length()).toBe(10)
})
test('should get the currentLength', () => {
const { buffer } = testInputBuffer('')
expect(buffer.currentLength()).toBe(0)
buffer.push('0123456789')
expect(buffer.currentLength()).toBe(10)
buffer.flush(4)
expect(buffer.currentLength()).toBe(10)
})
test('should flush', () => {
const { buffer } = testInputBuffer('0123456789')
expect(buffer.currentBufferSize()).toEqual(10)
buffer.flush(3)
expect(buffer.currentBufferSize()).toEqual(7)
})
test('should check whether we have reached the end', () => {
const { buffer } = testInputBuffer('0123456789')
buffer.close()
expect(buffer.isEnd(0)).toBe(false)
expect(buffer.isEnd(2)).toBe(false)
expect(buffer.isEnd(6)).toBe(false)
expect(buffer.isEnd(9)).toBe(false)
expect(buffer.isEnd(10)).toBe(true)
expect(buffer.isEnd(11)).toBe(true)
})
})
function testInputBuffer(text: string) {
const buffer = createInputBuffer()
buffer.push(text)
return { buffer }
}
```
## /src/streaming/buffer/InputBuffer.ts
```ts path="/src/streaming/buffer/InputBuffer.ts"
export interface InputBuffer {
push: (chunk: string) => void
flush: (position: number) => void
charAt: (index: number) => string
charCodeAt: (index: number) => number
substring: (start: number, end: number) => string
length: () => number
currentLength: () => number
currentBufferSize: () => number
isEnd: (index: number) => boolean
close: () => void
}
export function createInputBuffer(): InputBuffer {
let buffer = ''
let offset = 0
let currentLength = 0
let closed = false
function ensure(index: number) {
if (index < offset) {
throw new Error(`${indexOutOfRangeMessage} (index: ${index}, offset: ${offset})`)
}
if (index >= currentLength) {
if (!closed) {
throw new Error(`${indexOutOfRangeMessage} (index: ${index})`)
}
}
}
function push(chunk: string) {
buffer += chunk
currentLength += chunk.length
}
function flush(position: number) {
if (position > currentLength) {
return
}
buffer = buffer.substring(position - offset)
offset = position
}
function charAt(index: number): string {
ensure(index)
return buffer.charAt(index - offset)
}
function charCodeAt(index: number): number {
ensure(index)
return buffer.charCodeAt(index - offset)
}
function substring(start: number, end: number): string {
ensure(end - 1) // -1 because end is excluded
ensure(start)
return buffer.slice(start - offset, end - offset)
}
function length(): number {
if (!closed) {
throw new Error('Cannot get length: input is not yet closed')
}
return currentLength
}
function isEnd(index: number): boolean {
if (!closed) {
ensure(index)
}
return index >= currentLength
}
function close() {
closed = true
}
return {
push,
flush,
charAt,
charCodeAt,
substring,
length,
currentLength: () => currentLength,
currentBufferSize: () => buffer.length,
isEnd,
close
}
}
const indexOutOfRangeMessage = 'Index out of range, please configure a larger buffer size'
```
## /src/streaming/buffer/OutputBuffer.test.ts
```ts path="/src/streaming/buffer/OutputBuffer.test.ts"
import { describe, expect, test } from 'vitest'
import { createOutputBuffer } from './OutputBuffer'
describe('OutputBuffer', () => {
test('should write chunks into an output buffer', () => {
const { chunks, buffer } = testOutputBuffer({ chunkSize: 2, bufferSize: 2 })
buffer.push('0')
buffer.push('1')
buffer.push('2')
expect(chunks).toEqual([])
buffer.push('3')
buffer.push('4')
expect(chunks).toEqual(['01'])
buffer.push('5')
expect(chunks).toEqual(['01', '23'])
buffer.push('6')
buffer.flush()
expect(chunks).toEqual(['01', '23', '45', '6'])
})
test('should push to the output buffer', () => {
const { chunks, buffer } = testOutputBuffer()
buffer.push('test')
buffer.flush()
expect(chunks).toEqual(['test'])
})
test('should get current length', () => {
const { buffer } = testOutputBuffer({ chunkSize: 2, bufferSize: 2 })
buffer.push(':) ')
expect(buffer.length()).toEqual(3)
buffer.push('hello world')
expect(buffer.length()).toEqual(14)
})
test('should unshift', () => {
const { chunks, buffer } = testOutputBuffer()
buffer.push('hello world')
buffer.unshift(':) ')
buffer.flush()
expect(chunks).toEqual([':) hello world'])
})
test('should throw when it cannot unshift', () => {
const { chunks, buffer } = testOutputBuffer({ chunkSize: 2, bufferSize: 2 })
buffer.push('hello world')
expect(() => buffer.unshift(':) ')).toThrow(
/Cannot unshift: start of the output is already flushed from the buffer/
)
buffer.flush()
expect(chunks).toEqual(['he', 'll', 'o ', 'wo', 'rl', 'd'])
})
test('should remove without end', () => {
const { chunks, buffer } = testOutputBuffer()
buffer.push('hello world')
buffer.remove(5)
buffer.flush()
expect(chunks).toEqual(['hello'])
})
test('should remove with end', () => {
const { chunks, buffer } = testOutputBuffer()
buffer.push('how are you doing?')
buffer.remove(4, 8)
buffer.flush()
expect(chunks).toEqual(['how you doing?'])
})
// TODO: TEST stripLastOccurrence: (textToStrip: string, stripRemainingText?: boolean) => void
// TODO: TEST insertBeforeLastWhitespace: (textToInsert: string) => void
// TODO: TEST endsWithCommaOrNewline: () => boolean
// TODO: TEST length: () => number
// TODO: TEST close: () => void
})
function testOutputBuffer(options?: { chunkSize: number; bufferSize: number }) {
const chunks: string[] = []
const buffer = createOutputBuffer({
write: (chunk) => {
chunks.push(chunk)
},
...options
})
return { chunks, buffer }
}
```
## /src/streaming/buffer/OutputBuffer.ts
```ts path="/src/streaming/buffer/OutputBuffer.ts"
import { isWhitespace } from '../../utils/stringUtils.js'
export interface OutputBuffer {
push: (text: string) => void
unshift: (text: string) => void
remove: (start: number, end?: number) => void
insertAt: (index: number, text: string) => void
length: () => number
flush: () => void
stripLastOccurrence: (textToStrip: string, stripRemainingText?: boolean) => void
insertBeforeLastWhitespace: (textToInsert: string) => void
endsWithIgnoringWhitespace: (char: string) => boolean
}
export interface OutputBufferOptions {
write: (chunk: string) => void
chunkSize: number
bufferSize: number
}
export function createOutputBuffer({
write,
chunkSize,
bufferSize
}: OutputBufferOptions): OutputBuffer {
let buffer = ''
let offset = 0
function flushChunks(minSize = bufferSize) {
while (buffer.length >= minSize + chunkSize) {
const chunk = buffer.substring(0, chunkSize)
write(chunk)
offset += chunkSize
buffer = buffer.substring(chunkSize)
}
}
function flush() {
flushChunks(0)
if (buffer.length > 0) {
write(buffer)
offset += buffer.length
buffer = ''
}
}
function push(text: string) {
buffer += text
flushChunks()
}
function unshift(text: string) {
if (offset > 0) {
throw new Error(`Cannot unshift: ${flushedMessage}`)
}
buffer = text + buffer
flushChunks()
}
function remove(start: number, end?: number) {
if (start < offset) {
throw new Error(`Cannot remove: ${flushedMessage}`)
}
if (end !== undefined) {
buffer = buffer.substring(0, start - offset) + buffer.substring(end - offset)
} else {
buffer = buffer.substring(0, start - offset)
}
}
function insertAt(index: number, text: string) {
if (index < offset) {
throw new Error(`Cannot insert: ${flushedMessage}`)
}
buffer = buffer.substring(0, index - offset) + text + buffer.substring(index - offset)
}
function length(): number {
return offset + buffer.length
}
function stripLastOccurrence(textToStrip: string, stripRemainingText = false) {
const bufferIndex = buffer.lastIndexOf(textToStrip)
if (bufferIndex !== -1) {
if (stripRemainingText) {
buffer = buffer.substring(0, bufferIndex)
} else {
buffer =
buffer.substring(0, bufferIndex) + buffer.substring(bufferIndex + textToStrip.length)
}
}
}
function insertBeforeLastWhitespace(textToInsert: string) {
let bufferIndex = buffer.length // index relative to the start of the buffer, not taking `offset` into account
if (!isWhitespace(buffer, bufferIndex - 1)) {
// no trailing whitespaces
push(textToInsert)
return
}
while (isWhitespace(buffer, bufferIndex - 1)) {
bufferIndex--
}
if (bufferIndex <= 0) {
throw new Error(`Cannot insert: ${flushedMessage}`)
}
buffer = buffer.substring(0, bufferIndex) + textToInsert + buffer.substring(bufferIndex)
flushChunks()
}
function endsWithIgnoringWhitespace(char: string): boolean {
let i = buffer.length - 1
while (i > 0) {
if (char === buffer.charAt(i)) {
return true
}
if (!isWhitespace(buffer, i)) {
return false
}
i--
}
return false
}
return {
push,
unshift,
remove,
insertAt,
length,
flush,
stripLastOccurrence,
insertBeforeLastWhitespace,
endsWithIgnoringWhitespace
}
}
const flushedMessage = 'start of the output is already flushed from the buffer'
```
## /src/streaming/core.test.ts
```ts path="/src/streaming/core.test.ts"
import { describe, expect, test } from 'vitest'
import { jsonrepairCore } from './core'
describe('core', () => {
test('it should transform input in chunks', () => {
const { chunks, transform } = createCore({ bufferSize: 4, chunkSize: 2 })
transform.transform('[1')
transform.transform('2,')
transform.transform('3,')
transform.transform('4,')
expect(chunks).toEqual([])
transform.transform('5,')
expect(chunks).toEqual(['[1'])
transform.transform('6,')
expect(chunks).toEqual(['[1', '2,'])
transform.transform('7,')
expect(chunks).toEqual(['[1', '2,', '3,'])
transform.transform(']')
expect(chunks).toEqual(['[1', '2,', '3,'])
transform.flush()
expect(chunks).toEqual(['[1', '2,', '3,', '4,', '5,', '6,', '7]'])
})
test('it should throw an error when having a too small input buffer', () => {
const { transform } = createCore({ bufferSize: 4, chunkSize: 2 })
transform.transform('1234')
expect(() => {
transform.transform('56')
}).toThrow('Index out of range, please configure a larger buffer size (index: 6)')
})
test('it should throw an error when having a too small output buffer', () => {
// testing with ndjson, that has to insert a [ at the start of the document
const { transform } = createCore({ bufferSize: 4, chunkSize: 2 })
transform.transform('{"id":1}\n')
expect(() => {
transform.transform('{"id":2}\n')
}).toThrow('Cannot unshift: start of the output is already flushed from the buffer')
})
})
function createCore(options?: { chunkSize: number; bufferSize: number }) {
const chunks: string[] = []
const transform = jsonrepairCore({
bufferSize: options?.bufferSize,
chunkSize: options?.chunkSize,
onData: (chunk) => {
chunks.push(chunk)
}
})
return { chunks, transform }
}
```
## /src/streaming/core.ts
```ts path="/src/streaming/core.ts"
import { JSONRepairError } from '../utils/JSONRepairError.js'
import {
isControlCharacter,
isDelimiter,
isDigit,
isDoubleQuote,
isDoubleQuoteLike,
isFunctionNameChar,
isFunctionNameCharStart,
isHex,
isQuote,
isSingleQuote,
isSingleQuoteLike,
isSpecialWhitespace,
isStartOfValue,
isUnquotedStringDelimiter,
isValidStringCharacter,
isWhitespace,
isWhitespaceExceptNewline,
regexUrlChar,
regexUrlStart
} from '../utils/stringUtils.js'
import { createInputBuffer } from './buffer/InputBuffer.js'
import { createOutputBuffer } from './buffer/OutputBuffer.js'
import { Caret, createStack, StackType } from './stack.js'
const controlCharacters: { [key: string]: string } = {
'\b': '\\b',
'\f': '\\f',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t'
}
// map with all escape characters
const escapeCharacters: { [key: string]: string } = {
'"': '"',
'\\': '\\',
'/': '/',
b: '\b',
f: '\f',
n: '\n',
r: '\r',
t: '\t'
// note that \u is handled separately in parseString()
}
export interface JsonRepairCoreOptions {
onData: (chunk: string) => void
chunkSize?: number
bufferSize?: number
}
export interface JsonRepairCore {
transform: (chunk: string) => void
flush: () => void
}
export function jsonrepairCore({
onData,
bufferSize = 65536,
chunkSize = 65536
}: JsonRepairCoreOptions): JsonRepairCore {
const input = createInputBuffer()
const output = createOutputBuffer({
write: onData,
bufferSize,
chunkSize
})
let i = 0
let iFlushed = 0
const stack = createStack()
function flushInputBuffer() {
while (iFlushed < i - bufferSize - chunkSize) {
iFlushed += chunkSize
input.flush(iFlushed)
}
}
function transform(chunk: string) {
input.push(chunk)
while (i < input.currentLength() - bufferSize && parse()) {
// loop until there is nothing more to process
}
flushInputBuffer()
}
function flush() {
input.close()
while (parse()) {
// loop until there is nothing more to process
}
output.flush()
}
function parse(): boolean {
parseWhitespaceAndSkipComments()
switch (stack.type) {
case StackType.object: {
switch (stack.caret) {
case Caret.beforeKey:
return (
skipEllipsis() ||
parseObjectKey() ||
parseUnexpectedColon() ||
parseRepairTrailingComma() ||
parseRepairObjectEndOrComma()
)
case Caret.beforeValue:
return parseValue() || parseRepairMissingObjectValue()
case Caret.afterValue:
return parseObjectComma() || parseObjectEnd() || parseRepairObjectEndOrComma()
default:
return false
}
}
case StackType.array: {
switch (stack.caret) {
case Caret.beforeValue:
return (
skipEllipsis() || parseValue() || parseRepairTrailingComma() || parseRepairArrayEnd()
)
case Caret.afterValue:
return (
parseArrayComma() ||
parseArrayEnd() ||
parseRepairMissingComma() ||
parseRepairArrayEnd()
)
default:
return false
}
}
case StackType.ndJson: {
switch (stack.caret) {
case Caret.beforeValue:
return parseValue() || parseRepairTrailingComma()
case Caret.afterValue:
return parseArrayComma() || parseRepairMissingComma() || parseRepairNdJsonEnd()
default:
return false
}
}
case StackType.functionCall: {
switch (stack.caret) {
case Caret.beforeValue:
return parseValue()
case Caret.afterValue:
return parseFunctionCallEnd()
default:
return false
}
}
case StackType.root: {
switch (stack.caret) {
case Caret.beforeValue:
return parseRootStart()
case Caret.afterValue:
return parseRootEnd()
default:
return false
}
}
default:
return false
}
}
function parseValue(): boolean {
return (
parseObjectStart() ||
parseArrayStart() ||
parseString() ||
parseNumber() ||
parseKeywords() ||
parseRepairUnquotedString() ||
parseRepairRegex()
)
}
function parseObjectStart(): boolean {
if (parseCharacter('{')) {
parseWhitespaceAndSkipComments()
skipEllipsis()
if (skipCharacter(',')) {
parseWhitespaceAndSkipComments()
}
if (parseCharacter('}')) {
return stack.update(Caret.afterValue)
}
return stack.push(StackType.object, Caret.beforeKey)
}
return false
}
function parseArrayStart(): boolean {
if (parseCharacter('[')) {
parseWhitespaceAndSkipComments()
skipEllipsis()
if (skipCharacter(',')) {
parseWhitespaceAndSkipComments()
}
if (parseCharacter(']')) {
return stack.update(Caret.afterValue)
}
return stack.push(StackType.array, Caret.beforeValue)
}
return false
}
function parseRepairUnquotedString(): boolean {
let j = i
if (isFunctionNameCharStart(input.charAt(j))) {
while (!input.isEnd(j) && isFunctionNameChar(input.charAt(j))) {
j++
}
let k = j
while (isWhitespace(input, k)) {
k++
}
if (input.charAt(k) === '(') {
// repair a MongoDB function call like NumberLong("2")
// repair a JSONP function call like callback({...});
k++
i = k
return stack.push(StackType.functionCall, Caret.beforeValue)
}
}
j = findNextDelimiter(false, j)
if (j !== null) {
// test start of an url like "https://..." (this would be parsed as a comment)
if (input.charAt(j - 1) === ':' && regexUrlStart.test(input.substring(i, j + 2))) {
while (!input.isEnd(j) && regexUrlChar.test(input.charAt(j))) {
j++
}
}
const symbol = input.substring(i, j)
i = j
output.push(symbol === 'undefined' ? 'null' : JSON.stringify(symbol))
if (input.charAt(i) === '"') {
// we had a missing start quote, but now we encountered the end quote, so we can skip that one
i++
}
return stack.update(Caret.afterValue)
}
return false
}
function parseRepairRegex() {
if (input.charAt(i) === '/') {
const start = i
i++
while (!input.isEnd(i) && (input.charAt(i) !== '/' || input.charAt(i - 1) === '\\')) {
i++
}
i++
output.push(JSON.stringify(input.substring(start, i)))
return stack.update(Caret.afterValue)
}
}
function parseRepairMissingObjectValue(): boolean {
// repair missing object value
output.push('null')
return stack.update(Caret.afterValue)
}
function parseRepairTrailingComma(): boolean {
// repair trailing comma
if (output.endsWithIgnoringWhitespace(',')) {
output.stripLastOccurrence(',')
return stack.update(Caret.afterValue)
}
return false
}
function parseUnexpectedColon(): boolean {
if (input.charAt(i) === ':') {
throwObjectKeyExpected()
}
return false
}
function parseUnexpectedEnd(): boolean {
if (input.isEnd(i)) {
throwUnexpectedEnd()
} else {
throwUnexpectedCharacter()
}
return false
}
function parseObjectKey(): boolean {
const parsedKey = parseString() || parseUnquotedKey()
if (parsedKey) {
parseWhitespaceAndSkipComments()
if (parseCharacter(':')) {
// expect a value after the :
return stack.update(Caret.beforeValue)
}
const truncatedText = input.isEnd(i)
if (isStartOfValue(input.charAt(i)) || truncatedText) {
// repair missing colon
output.insertBeforeLastWhitespace(':')
return stack.update(Caret.beforeValue)
}
throwColonExpected()
}
return false
}
function parseObjectComma(): boolean {
if (parseCharacter(',')) {
return stack.update(Caret.beforeKey)
}
return false
}
function parseObjectEnd(): boolean {
if (parseCharacter('}')) {
return stack.pop()
}
return false
}
function parseRepairObjectEndOrComma(): true {
// repair missing object end and trailing comma
if (input.charAt(i) === '{') {
output.stripLastOccurrence(',')
output.insertBeforeLastWhitespace('}')
return stack.pop()
}
// repair missing comma
if (!input.isEnd(i) && isStartOfValue(input.charAt(i))) {
output.insertBeforeLastWhitespace(',')
return stack.update(Caret.beforeKey)
}
// repair missing closing brace
output.insertBeforeLastWhitespace('}')
return stack.pop()
}
function parseArrayComma(): boolean {
if (parseCharacter(',')) {
return stack.update(Caret.beforeValue)
}
return false
}
function parseArrayEnd(): boolean {
if (parseCharacter(']')) {
return stack.pop()
}
return false
}
function parseRepairMissingComma(): boolean {
// repair missing comma
if (!input.isEnd(i) && isStartOfValue(input.charAt(i))) {
output.insertBeforeLastWhitespace(',')
return stack.update(Caret.beforeValue)
}
return false
}
function parseRepairArrayEnd(): true {
// repair missing closing bracket
output.insertBeforeLastWhitespace(']')
return stack.pop()
}
function parseRepairNdJsonEnd(): boolean {
if (input.isEnd(i)) {
output.push('\n]')
return stack.pop()
}
throwUnexpectedEnd()
return false // just to make TS happy
}
function parseFunctionCallEnd(): true {
if (skipCharacter(')')) {
skipCharacter(';')
}
return stack.pop()
}
function parseRootStart(): boolean {
parseMarkdownCodeBlock(['\`\`\`', '[\`\`\`', '{\`\`\`'])
return parseValue() || parseUnexpectedEnd()
}
function parseRootEnd(): boolean {
parseMarkdownCodeBlock(['\`\`\`', '\`\`\`]', '\`\`\`}'])
const parsedComma = parseCharacter(',')
parseWhitespaceAndSkipComments()
if (
isStartOfValue(input.charAt(i)) &&
(output.endsWithIgnoringWhitespace(',') || output.endsWithIgnoringWhitespace('\n'))
) {
// start of a new value after end of the root level object: looks like
// newline delimited JSON -> turn into a root level array
if (!parsedComma) {
// repair missing comma
output.insertBeforeLastWhitespace(',')
}
output.unshift('[\n')
return stack.push(StackType.ndJson, Caret.beforeValue)
}
if (parsedComma) {
// repair: remove trailing comma
output.stripLastOccurrence(',')
return stack.update(Caret.afterValue)
}
// repair redundant end braces and brackets
while (input.charAt(i) === '}' || input.charAt(i) === ']') {
i++
parseWhitespaceAndSkipComments()
}
if (!input.isEnd(i)) {
throwUnexpectedCharacter()
}
return false
}
function parseWhitespaceAndSkipComments(skipNewline = true): boolean {
const start = i
let changed = parseWhitespace(skipNewline)
do {
changed = parseComment()
if (changed) {
changed = parseWhitespace(skipNewline)
}
} while (changed)
return i > start
}
function parseWhitespace(skipNewline: boolean): boolean {
const _isWhiteSpace = skipNewline ? isWhitespace : isWhitespaceExceptNewline
let whitespace = ''
while (true) {
if (_isWhiteSpace(input, i)) {
whitespace += input.charAt(i)
i++
} else if (isSpecialWhitespace(input, i)) {
// repair special whitespace
whitespace += ' '
i++
} else {
break
}
}
if (whitespace.length > 0) {
output.push(whitespace)
return true
}
return false
}
function parseComment(): boolean {
// find a block comment '/* ... */'
if (input.charAt(i) === '/' && input.charAt(i + 1) === '*') {
// repair block comment by skipping it
while (!input.isEnd(i) && !atEndOfBlockComment(i)) {
i++
}
i += 2
return true
}
// find a line comment '// ...'
if (input.charAt(i) === '/' && input.charAt(i + 1) === '/') {
// repair line comment by skipping it
while (!input.isEnd(i) && input.charAt(i) !== '\n') {
i++
}
return true
}
return false
}
function parseMarkdownCodeBlock(blocks: string[]): boolean {
// find and skip over a Markdown fenced code block:
// \`\`\` ... \`\`\`
// or
// \`\`\`json ... \`\`\`
if (skipMarkdownCodeBlock(blocks)) {
if (isFunctionNameCharStart(input.charAt(i))) {
// strip the optional language specifier like "json"
while (!input.isEnd(i) && isFunctionNameChar(input.charAt(i))) {
i++
}
}
parseWhitespaceAndSkipComments()
return true
}
return false
}
function skipMarkdownCodeBlock(blocks: string[]): boolean {
for (const block of blocks) {
const end = i + block.length
if (input.substring(i, end) === block) {
i = end
return true
}
}
return false
}
function parseCharacter(char: string): boolean {
if (input.charAt(i) === char) {
output.push(input.charAt(i))
i++
return true
}
return false
}
function skipCharacter(char: string): boolean {
if (input.charAt(i) === char) {
i++
return true
}
return false
}
function skipEscapeCharacter(): boolean {
return skipCharacter('\\')
}
/**
* Skip ellipsis like "[1,2,3,...]" or "[1,2,3,...,9]" or "[...,7,8,9]"
* or a similar construct in objects.
*/
function skipEllipsis(): boolean {
parseWhitespaceAndSkipComments()
if (input.charAt(i) === '.' && input.charAt(i + 1) === '.' && input.charAt(i + 2) === '.') {
// repair: remove the ellipsis (three dots) and optionally a comma
i += 3
parseWhitespaceAndSkipComments()
skipCharacter(',')
return true
}
return false
}
/**
* Parse a string enclosed by double quotes "...". Can contain escaped quotes
* Repair strings enclosed in single quotes or special quotes
* Repair an escaped string
*
* The function can run in two stages:
* - First, it assumes the string has a valid end quote
* - If it turns out that the string does not have a valid end quote followed
* by a delimiter (which should be the case), the function runs again in a
* more conservative way, stopping the string at the first next delimiter
* and fixing the string by inserting a quote there, or stopping at a
* stop index detected in the first iteration.
*/
function parseString(stopAtDelimiter = false, stopAtIndex = -1): boolean {
let skipEscapeChars = input.charAt(i) === '\\'
if (skipEscapeChars) {
// repair: remove the first escape character
i++
skipEscapeChars = true
}
if (isQuote(input.charAt(i))) {
// double quotes are correct JSON,
// single quotes come from JavaScript for example, we assume it will have a correct single end quote too
// otherwise, we will match any double-quote-like start with a double-quote-like end,
// or any single-quote-like start with a single-quote-like end
const isEndQuote = isDoubleQuote(input.charAt(i))
? isDoubleQuote
: isSingleQuote(input.charAt(i))
? isSingleQuote
: isSingleQuoteLike(input.charAt(i))
? isSingleQuoteLike
: isDoubleQuoteLike
const iBefore = i
const oBefore = output.length()
output.push('"')
i++
while (true) {
if (input.isEnd(i)) {
// end of text, we have a missing quote somewhere
const iPrev = prevNonWhitespaceIndex(i - 1)
if (!stopAtDelimiter && isDelimiter(input.charAt(iPrev))) {
// if the text ends with a delimiter, like ["hello],
// so the missing end quote should be inserted before this delimiter
// retry parsing the string, stopping at the first next delimiter
i = iBefore
output.remove(oBefore)
return parseString(true)
}
// repair missing quote
output.insertBeforeLastWhitespace('"')
return stack.update(Caret.afterValue)
}
if (i === stopAtIndex) {
// use the stop index detected in the first iteration, and repair end quote
output.insertBeforeLastWhitespace('"')
return stack.update(Caret.afterValue)
}
if (isEndQuote(input.charAt(i))) {
// end quote
// let us check what is before and after the quote to verify whether this is a legit end quote
const iQuote = i
const oQuote = output.length()
output.push('"')
i++
parseWhitespaceAndSkipComments(false)
if (
stopAtDelimiter ||
input.isEnd(i) ||
isDelimiter(input.charAt(i)) ||
isQuote(input.charAt(i)) ||
isDigit(input.charAt(i))
) {
// The quote is followed by the end of the text, a delimiter, or a next value
// so the quote is indeed the end of the string
parseConcatenatedString()
return stack.update(Caret.afterValue)
}
const iPrevChar = prevNonWhitespaceIndex(iQuote - 1)
const prevChar = input.charAt(iPrevChar)
if (prevChar === ',') {
// A comma followed by a quote, like '{"a":"b,c,"d":"e"}'.
// We assume that the quote is a start quote, and that the end quote
// should have been located right before the comma but is missing.
i = iBefore
output.remove(oBefore)
return parseString(false, iPrevChar)
}
if (isDelimiter(prevChar)) {
// This is not the right end quote: it is preceded by a delimiter,
// and NOT followed by a delimiter. So, there is an end quote missing
// parse the string again and then stop at the first next delimiter
i = iBefore
output.remove(oBefore)
return parseString(true)
}
// revert to right after the quote but before any whitespace, and continue parsing the string
output.remove(oQuote + 1)
i = iQuote + 1
// repair unescaped quote
output.insertAt(oQuote, '\\')
} else if (stopAtDelimiter && isUnquotedStringDelimiter(input.charAt(i))) {
// we're in the mode to stop the string at the first delimiter
// because there is an end quote missing
// test start of an url like "https://..." (this would be parsed as a comment)
if (
input.charAt(i - 1) === ':' &&
regexUrlStart.test(input.substring(iBefore + 1, i + 2))
) {
while (!input.isEnd(i) && regexUrlChar.test(input.charAt(i))) {
output.push(input.charAt(i))
i++
}
}
// repair missing quote
output.insertBeforeLastWhitespace('"')
parseConcatenatedString()
return stack.update(Caret.afterValue)
} else if (input.charAt(i) === '\\') {
// handle escaped content like \n or \u2605
const char = input.charAt(i + 1)
const escapeChar = escapeCharacters[char]
if (escapeChar !== undefined) {
output.push(input.substring(i, i + 2))
i += 2
} else if (char === 'u') {
let j = 2
while (j < 6 && isHex(input.charAt(i + j))) {
j++
}
if (j === 6) {
output.push(input.substring(i, i + 6))
i += 6
} else if (input.isEnd(i + j)) {
// repair invalid or truncated unicode char at the end of the text
// by removing the unicode char and ending the string here
i += j
} else {
throwInvalidUnicodeCharacter()
}
} else if (char === '\n') {
// repair a backslash escaped newline (like in Bash scripts)
output.push('\\n')
i += 2
} else {
// repair invalid escape character: remove it
output.push(char)
i += 2
}
} else {
// handle regular characters
const char = input.charAt(i)
if (char === '"' && input.charAt(i - 1) !== '\\') {
// repair unescaped double quote
output.push(`\\${char}`)
i++
} else if (isControlCharacter(char)) {
// unescaped control character
output.push(controlCharacters[char])
i++
} else {
if (!isValidStringCharacter(char)) {
throwInvalidCharacter(char)
}
output.push(char)
i++
}
}
if (skipEscapeChars) {
// repair: skipped escape character (nothing to do)
skipEscapeCharacter()
}
}
}
return false
}
/**
* Repair concatenated strings like "hello" + "world", change this into "helloworld"
*/
function parseConcatenatedString(): boolean {
let parsed = false
parseWhitespaceAndSkipComments()
while (input.charAt(i) === '+') {
parsed = true
i++
parseWhitespaceAndSkipComments()
// repair: remove the end quote of the first string
output.stripLastOccurrence('"', true)
const start = output.length()
const parsedStr = parseString()
if (parsedStr) {
// repair: remove the start quote of the second string
output.remove(start, start + 1)
} else {
// repair: remove the + because it is not followed by a string
output.insertBeforeLastWhitespace('"')
}
}
return parsed
}
/**
* Parse a number like 2.4 or 2.4e6
*/
function parseNumber(): boolean {
const start = i
if (input.charAt(i) === '-') {
i++
if (atEndOfNumber()) {
repairNumberEndingWithNumericSymbol(start)
return stack.update(Caret.afterValue)
}
if (!isDigit(input.charAt(i))) {
i = start
return false
}
}
// Note that in JSON leading zeros like "00789" are not allowed.
// We will allow all leading zeros here though and at the end of parseNumber
// check against trailing zeros and repair that if needed.
// Leading zeros can have meaning, so we should not clear them.
while (isDigit(input.charAt(i))) {
i++
}
if (input.charAt(i) === '.') {
i++
if (atEndOfNumber()) {
repairNumberEndingWithNumericSymbol(start)
return stack.update(Caret.afterValue)
}
if (!isDigit(input.charAt(i))) {
i = start
return false
}
while (isDigit(input.charAt(i))) {
i++
}
}
if (input.charAt(i) === 'e' || input.charAt(i) === 'E') {
i++
if (input.charAt(i) === '-' || input.charAt(i) === '+') {
i++
}
if (atEndOfNumber()) {
repairNumberEndingWithNumericSymbol(start)
return stack.update(Caret.afterValue)
}
if (!isDigit(input.charAt(i))) {
i = start
return false
}
while (isDigit(input.charAt(i))) {
i++
}
}
// if we're not at the end of the number by this point, allow this to be parsed as another type
if (!atEndOfNumber()) {
i = start
return false
}
if (i > start) {
// repair a number with leading zeros like "00789"
const num = input.substring(start, i)
const hasInvalidLeadingZero = /^0\d/.test(num)
output.push(hasInvalidLeadingZero ? `"${num}"` : num)
return stack.update(Caret.afterValue)
}
return false
}
/**
* Parse keywords true, false, null
* Repair Python keywords True, False, None
*/
function parseKeywords(): boolean {
return (
parseKeyword('true', 'true') ||
parseKeyword('false', 'false') ||
parseKeyword('null', 'null') ||
// repair Python keywords True, False, None
parseKeyword('True', 'true') ||
parseKeyword('False', 'false') ||
parseKeyword('None', 'null')
)
}
function parseKeyword(name: string, value: string): boolean {
if (input.substring(i, i + name.length) === name) {
output.push(value)
i += name.length
return stack.update(Caret.afterValue)
}
return false
}
function parseUnquotedKey(): boolean {
let end = findNextDelimiter(true, i)
if (end !== null) {
// first, go back to prevent getting trailing whitespaces in the string
while (isWhitespace(input, end - 1) && end > i) {
end--
}
const symbol = input.substring(i, end)
output.push(JSON.stringify(symbol))
i = end
if (input.charAt(i) === '"') {
// we had a missing start quote, but now we encountered the end quote, so we can skip that one
i++
}
return stack.update(Caret.afterValue) // we do not have a state Caret.afterKey, therefore we use afterValue here
}
return false
}
function findNextDelimiter(isKey: boolean, start: number): number | null {
// note that the symbol can end with whitespaces: we stop at the next delimiter
// also, note that we allow strings to contain a slash / in order to support repairing regular expressions
let j = start
while (
!input.isEnd(j) &&
!isUnquotedStringDelimiter(input.charAt(j)) &&
!isQuote(input.charAt(j)) &&
(!isKey || input.charAt(j) !== ':')
) {
j++
}
return j > i ? j : null
}
function prevNonWhitespaceIndex(start: number): number {
let prev = start
while (prev > 0 && isWhitespace(input, prev)) {
prev--
}
return prev
}
function atEndOfNumber() {
return input.isEnd(i) || isDelimiter(input.charAt(i)) || isWhitespace(input, i)
}
function repairNumberEndingWithNumericSymbol(start: number) {
// repair numbers cut off at the end
// this will only be called when we end after a '.', '-', or 'e' and does not
// change the number more than it needs to make it valid JSON
output.push(`${input.substring(start, i)}0`)
}
function throwInvalidCharacter(char: string) {
throw new JSONRepairError(`Invalid character ${JSON.stringify(char)}`, i)
}
function throwUnexpectedCharacter() {
throw new JSONRepairError(`Unexpected character ${JSON.stringify(input.charAt(i))}`, i)
}
function throwUnexpectedEnd() {
throw new JSONRepairError('Unexpected end of json string', i)
}
function throwObjectKeyExpected() {
throw new JSONRepairError('Object key expected', i)
}
function throwColonExpected() {
throw new JSONRepairError('Colon expected', i)
}
function throwInvalidUnicodeCharacter() {
const chars = input.substring(i, i + 6)
throw new JSONRepairError(`Invalid unicode character "${chars}"`, i)
}
function atEndOfBlockComment(i: number) {
return input.charAt(i) === '*' && input.charAt(i + 1) === '/'
}
return {
transform,
flush
}
}
```
## /src/streaming/stack.ts
```ts path="/src/streaming/stack.ts"
export enum Caret {
beforeValue = 'beforeValue',
afterValue = 'afterValue',
beforeKey = 'beforeKey'
}
export enum StackType {
root = 'root',
object = 'object',
array = 'array',
ndJson = 'ndJson',
functionCall = 'dataType'
}
export function createStack() {
const stack: StackType[] = [StackType.root]
let caret = Caret.beforeValue
return {
get type() {
return last(stack)
},
get caret() {
return caret
},
pop(): true {
stack.pop()
caret = Caret.afterValue
return true
},
push(type: StackType, newCaret: Caret): true {
stack.push(type)
caret = newCaret
return true
},
update(newCaret: Caret): true {
caret = newCaret
return true
}
}
}
function last<T>(array: T[]): T | undefined {
return array[array.length - 1]
}
```
## /src/streaming/stream.test.ts
```ts path="/src/streaming/stream.test.ts"
import { Readable, type Transform } from 'node:stream'
import { describe, expect, test } from 'vitest'
import { jsonrepairTransform } from './stream'
describe('stream', () => {
test('should create and pipe a jsonrepair transform', async () => {
const input = new Readable()
input.push("{name: 'John'}")
input.push(null)
const output = input.pipe(jsonrepairTransform())
const result = await streamToChunks(output)
expect(result).toEqual(['{"name": "John"}'])
})
test('should configure chunk size', async () => {
const input = new Readable()
input.push("{name: 'John'}")
input.push(null)
const output = input.pipe(jsonrepairTransform({ chunkSize: 4 }))
const result = await streamToChunks(output)
expect(result).toEqual(['{"na', 'me":', ' "Jo', 'hn"}'])
})
test('should configure buffer size, should throw error', async () => {
return new Promise<void>((resolve) => {
const input = new Readable()
input.push("{name: 'John', }")
input.push(null)
const output = input.pipe(jsonrepairTransform({ chunkSize: 4, bufferSize: 2 }))
input.on('error', (err) => {
console.log('Error', err)
})
streamToChunks(output)
.then(() => {
throw new Error('Should not succeed')
})
.catch((err) => {
expect(err.toString()).toEqual(
'Error: Cannot insert: start of the output is already flushed from the buffer'
)
resolve()
})
})
})
})
function streamToChunks(stream: Transform): Promise<string[]> {
return new Promise((resolve, reject) => {
const chunks: string[] = []
stream.on('data', (chunk) => chunks.push(chunk.toString()))
stream.on('error', (err) => reject(err))
stream.on('end', () => resolve(chunks))
})
}
```
## /src/streaming/stream.ts
```ts path="/src/streaming/stream.ts"
import { Transform } from 'node:stream'
import { jsonrepairCore } from './core.js'
export interface JsonRepairTransformOptions {
chunkSize?: number
bufferSize?: number
}
export function jsonrepairTransform(options?: JsonRepairTransformOptions): Transform {
const repair = jsonrepairCore({
onData: (chunk) => transform.push(chunk),
bufferSize: options?.bufferSize,
chunkSize: options?.chunkSize
})
const transform = new Transform({
transform(chunk, _encoding, callback) {
try {
repair.transform(chunk.toString())
} catch (err) {
this.emit('error', err)
} finally {
callback()
}
},
flush(callback) {
try {
repair.flush()
} catch (err) {
this.emit('error', err)
} finally {
callback()
}
}
})
return transform
}
```
## /src/utils/JSONRepairError.ts
```ts path="/src/utils/JSONRepairError.ts"
export class JSONRepairError extends Error {
position: number
constructor(message: string, position: number) {
super(`${message} at position ${position}`)
this.position = position
}
}
```
## /src/utils/stringUtils.ts
```ts path="/src/utils/stringUtils.ts"
const codeSpace = 0x20 // " "
const codeNewline = 0xa // "\n"
const codeTab = 0x9 // "\t"
const codeReturn = 0xd // "\r"
// unicode spaces: https://jkorpela.fi/chars/spaces.html
const codeNonBreakingSpace = 0x00a0
const codeMongolianVowelSeparator = 0x180e
const codeEnQuad = 0x2000
const codeZeroWidthSpace = 0x200b
const codeNarrowNoBreakSpace = 0x202f
const codeMediumMathematicalSpace = 0x205f
const codeIdeographicSpace = 0x3000
const codeZeroWidthNoBreakSpace = 0xfeff
export function isHex(char: string): boolean {
return /^[0-9A-Fa-f]$/.test(char)
}
export function isDigit(char: string): boolean {
return char >= '0' && char <= '9'
}
export function isValidStringCharacter(char: string): boolean {
// note that the valid range is between \u{0020} and \u{10ffff},
// but in JavaScript it is not possible to create a code point larger than
// \u{10ffff}, so there is no need to test for that here.
return char >= '\u0020'
}
export function isDelimiter(char: string): boolean {
return ',:[]/{}()\n+'.includes(char)
}
export function isFunctionNameCharStart(char: string) {
return (
(char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || char === '_' || char === '{{contextString}}#39;
)
}
export function isFunctionNameChar(char: string) {
return (
(char >= 'a' && char <= 'z') ||
(char >= 'A' && char <= 'Z') ||
char === '_' ||
char === '{{contextString}}#39; ||
(char >= '0' && char <= '9')
)
}
// matches "https://" and other schemas
export const regexUrlStart = /^(http|https|ftp|mailto|file|data|irc):\/\/$/
// matches all valid URL characters EXCEPT "[", "]", and ",", since that are important JSON delimiters
export const regexUrlChar = /^[A-Za-z0-9-._~:/?#@!{{contextString}}amp;'()*+;=]$/
export function isUnquotedStringDelimiter(char: string): boolean {
return ',[]/{}\n+'.includes(char)
}
export function isStartOfValue(char: string): boolean {
return isQuote(char) || regexStartOfValue.test(char)
}
// alpha, number, minus, or opening bracket or brace
const regexStartOfValue = /^[[{\w-]$/
export function isControlCharacter(char: string) {
return char === '\n' || char === '\r' || char === '\t' || char === '\b' || char === '\f'
}
export interface Text {
charCodeAt: (index: number) => number
}
/**
* Check if the given character is a whitespace character like space, tab, or
* newline
*/
export function isWhitespace(text: Text, index: number): boolean {
const code = text.charCodeAt(index)
return code === codeSpace || code === codeNewline || code === codeTab || code === codeReturn
}
/**
* Check if the given character is a whitespace character like space or tab,
* but NOT a newline
*/
export function isWhitespaceExceptNewline(text: Text, index: number): boolean {
const code = text.charCodeAt(index)
return code === codeSpace || code === codeTab || code === codeReturn
}
/**
* Check if the given character is a special whitespace character, some
* unicode variant
*/
export function isSpecialWhitespace(text: Text, index: number): boolean {
const code = text.charCodeAt(index)
return (
code === codeNonBreakingSpace ||
code === codeMongolianVowelSeparator ||
(code >= codeEnQuad && code <= codeZeroWidthSpace) ||
code === codeNarrowNoBreakSpace ||
code === codeMediumMathematicalSpace ||
code === codeIdeographicSpace ||
code === codeZeroWidthNoBreakSpace
)
}
/**
* Test whether the given character is a quote or double quote character.
* Also tests for special variants of quotes.
*/
export function isQuote(char: string): boolean {
// the first check double quotes, since that occurs most often
return isDoubleQuoteLike(char) || isSingleQuoteLike(char)
}
/**
* Test whether the given character is a double quote character.
* Also tests for special variants of double quotes.
*/
export function isDoubleQuoteLike(char: string): boolean {
return char === '"' || char === '\u201c' || char === '\u201d'
}
/**
* Test whether the given character is a double quote character.
* Does NOT test for special variants of double quotes.
*/
export function isDoubleQuote(char: string): boolean {
return char === '"'
}
/**
* Test whether the given character is a single quote character.
* Also tests for special variants of single quotes.
*/
export function isSingleQuoteLike(char: string): boolean {
return (
char === "'" || char === '\u2018' || char === '\u2019' || char === '\u0060' || char === '\u00b4'
)
}
/**
* Test whether the given character is a single quote character.
* Does NOT test for special variants of single quotes.
*/
export function isSingleQuote(char: string): boolean {
return char === "'"
}
/**
* Strip last occurrence of textToStrip from text
*/
export function stripLastOccurrence(
text: string,
textToStrip: string,
stripRemainingText = false
): string {
const index = text.lastIndexOf(textToStrip)
return index !== -1
? text.substring(0, index) + (stripRemainingText ? '' : text.substring(index + 1))
: text
}
export function insertBeforeLastWhitespace(text: string, textToInsert: string): string {
let index = text.length
if (!isWhitespace(text, index - 1)) {
// no trailing whitespaces
return text + textToInsert
}
while (isWhitespace(text, index - 1)) {
index--
}
return text.substring(0, index) + textToInsert + text.substring(index)
}
export function removeAtIndex(text: string, start: number, count: number) {
return text.substring(0, start) + text.substring(start + count)
}
/**
* Test whether a string ends with a newline or comma character and optional whitespace
*/
export function endsWithCommaOrNewline(text: string): boolean {
return /[,\n][ \t\r]*$/.test(text)
}
```
## /test-lib/apps/cjsApp.cjs
```cjs path="/test-lib/apps/cjsApp.cjs"
const { jsonrepair } = require('../..')
console.log(jsonrepair("{name: 'John'}"))
```
## /test-lib/apps/cjsAppStreaming.cjs
```cjs path="/test-lib/apps/cjsAppStreaming.cjs"
const { jsonrepairTransform } = require('../../lib/cjs/stream.js')
const { Readable } = require('node:stream')
const input = new Readable()
input.push("{name: 'John'}")
input.push(null)
input.pipe(jsonrepairTransform()).pipe(process.stdout)
```
## /test-lib/apps/esmApp.mjs
```mjs path="/test-lib/apps/esmApp.mjs"
import { jsonrepair } from '../../lib/esm/index.js'
console.log(jsonrepair("{name: 'John'}"))
```
## /test-lib/apps/esmAppStreaming.mjs
```mjs path="/test-lib/apps/esmAppStreaming.mjs"
import { Readable } from 'node:stream'
import { jsonrepairTransform } from '../../lib/esm/stream.js'
const input = new Readable()
input.push("{name: 'John'}")
input.push(null)
input.pipe(jsonrepairTransform()).pipe(process.stdout)
```
## /test-lib/apps/esmBrowserApp.html
```html path="/test-lib/apps/esmBrowserApp.html"
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Simple JSON Repair</title>
</head>
<body>
<div id="invalid"></div>
<div id="repaired"></div>
<script type="module">
import { jsonrepair } from '../../lib/esm/index.js'
const invalidJson = '{name: \'John\'}'
const repairedJson = jsonrepair(invalidJson)
document.getElementById('invalid').innerHTML = `Invalid JSON:${invalidJson}`
document.getElementById('repaired').innerHTML = `Repaired JSON:${repairedJson}`
</script>
</body>
</html>
```
## /test-lib/apps/umdApp.cjs
```cjs path="/test-lib/apps/umdApp.cjs"
const { jsonrepair } = require('../../lib/umd/jsonrepair.js')
console.log(jsonrepair("{name: 'John'}"))
```
## /test-lib/apps/umdAppMin.cjs
```cjs path="/test-lib/apps/umdAppMin.cjs"
const { jsonrepair } = require('../../lib/umd/jsonrepair.min.js')
console.log(jsonrepair("{name: 'John'}"))
```
## /test-lib/apps/umdBrowserApp.html
```html path="/test-lib/apps/umdBrowserApp.html"
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Simple JSON Repair</title>
<script src="../../lib/umd/jsonrepair.js"></script>
</head>
<body>
<div id="invalid"></div>
<div id="repaired"></div>
<script>
const invalidJson = '{name: \'John\'}'
const repairedJson = JSONRepair.jsonrepair(invalidJson)
document.getElementById('invalid').innerHTML = `Invalid JSON:${invalidJson}`
document.getElementById('repaired').innerHTML = `Repaired JSON:${repairedJson}`
</script>
</body>
</html>
```
## /test-lib/cli.test.js
```js path="/test-lib/cli.test.js"
// Only use native node.js API's and references to ./lib here, this file is not transpiled!
import cp from 'node:child_process'
import { copyFileSync, existsSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, beforeEach, describe, expect, test } from 'vitest'
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
describe('command line interface', () => {
const binFile = join(__dirname, '..', 'bin', 'cli.js')
const inputFile = join(__dirname, 'data', 'invalid.json')
const replaceFile = join(__dirname, 'output', 'replace.json')
const outputFile = join(__dirname, 'output', 'repaired.json')
const largeFile = join(__dirname, 'output', 'large.json')
beforeEach(() => {
if (existsSync(outputFile)) {
rmSync(outputFile)
}
copyFileSync(inputFile, replaceFile)
})
afterEach(() => {
if (existsSync(outputFile)) {
rmSync(outputFile)
}
if (existsSync(replaceFile)) {
rmSync(replaceFile)
}
})
test('should write to the console', async () => {
const result = await run(`node "${binFile}" "${inputFile}"`)
expect(stripNewlines(result)).toBe('{"hello":"world"}')
})
test('should write output to a file', async () => {
const result = await run(`node "${binFile}" "${inputFile}" > "${outputFile}"`)
expect(result).toBe('')
const content = String(readFileSync(outputFile))
expect(stripNewlines(content)).toBe('{"hello":"world"}')
})
test('should replace a file', async () => {
const result = await run(`node ${binFile} "${replaceFile}" --overwrite`)
expect(result).toBe('')
const content = String(readFileSync(replaceFile))
expect(stripNewlines(content)).toBe('{"hello":"world"}')
})
test('should configure buffer size', async () => {
// create a document that is larger than the 64K chunk size of the library
const largeDoc = new Array(10_000).fill('test test test ')
const str = `{"a": ["${largeDoc.join('')}`
expect(str.length).toBeGreaterThan(65536)
writeFileSync(largeFile, str)
await expect(() => {
return run(`node ${binFile} "${largeFile}" --buffer 2`)
}).rejects.toThrow(
'Error: Index out of range, please configure a larger buffer size (index: 65536)'
)
unlinkSync(largeFile)
})
test('should throw an error in case of an invalid buffer size', async () => {
await expect(() => {
return run(`node ${binFile} "${inputFile}" --buffer FOO`)
}).rejects.toThrow('Error: Buffer size "FOO" not recognized')
})
})
function run(command) {
return new Promise((resolve, reject) => {
cp.exec(command, (error, result) => {
if (error) {
reject(error)
} else {
resolve(result)
}
})
})
}
function stripNewlines(text) {
return text.replace(/[\n\r]/g, '')
}
```
## /test-lib/data/invalid.json
```json path="/test-lib/data/invalid.json"
{hello:world}
```
## /test-lib/lib.test.js
```js path="/test-lib/lib.test.js"
import cp from 'node:child_process'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, test } from 'vitest'
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)
describe('lib', () => {
test('should load the library using CJS', async () => {
const filename = join(__dirname, 'apps/cjsApp.cjs')
const result = await run(`node ${filename}`)
expect(result).toBe('{"name": "John"}\n')
})
test('should load the library using CJS (streaming)', async () => {
const filename = join(__dirname, 'apps/cjsAppStreaming.cjs')
const result = await run(`node ${filename}`)
expect(result).toBe('{"name": "John"}')
})
test('should load the library using ESM', async () => {
const filename = join(__dirname, 'apps/esmApp.mjs')
const result = await run(`node ${filename}`)
expect(result).toBe('{"name": "John"}\n')
})
test('should load the library using ESM (streaming)', async () => {
const filename = join(__dirname, 'apps/esmAppStreaming.mjs')
const result = await run(`node ${filename}`)
expect(result).toBe('{"name": "John"}')
})
test('should load the library using UMD bundle', async () => {
const filename = join(__dirname, 'apps/umdApp.cjs')
const result = await run(`node ${filename}`)
expect(result).toBe('{"name": "John"}\n')
})
test('should load the library using minified UMD bundle', async () => {
const filename = join(__dirname, 'apps/umdAppMin.cjs')
const result = await run(`node ${filename}`)
expect(result).toBe('{"name": "John"}\n')
})
})
function run(command) {
return new Promise((resolve, reject) => {
cp.exec(command, (error, result) => {
if (error) {
reject(error)
} else {
resolve(result)
}
})
})
}
```
## /test-lib/output/.gitignore
```gitignore path="/test-lib/output/.gitignore"
*
```
## /test-lib/test.html
```html path="/test-lib/test.html"
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>jsonrepair test performance</title>
</head>
<body>
<div class="app">
<h1>jsonrepair test performance</h1>
<p>(check the developer console output)</p>
<div class="info">
<input id="loadFile" type="file">
</div>
</div>
<script type="module">
import { jsonrepair } from '../lib/esm/index.js'
function repairIt (text) {
console.time('repair')
const repaired = jsonrepair(text)
console.timeEnd('repair')
console.log('done, length:', {
textLength: text.length,
repairedLength: repaired.length,
changed: repaired !== text
})
}
document.getElementById('loadFile').onchange = function loadFile(event) {
console.log('loadFile', event.target.files)
console.time('load file')
const reader = new FileReader()
const file = event.target.files[0]
reader.onload = (event) => {
console.timeEnd('load file')
const text = event.target.result
repairIt(text)
}
reader.readAsText(file)
}
</script>
</body>
</html>
```
## /tools/benchmark/run.mjs
```mjs path="/tools/benchmark/run.mjs"
import assert from 'node:assert'
import { Bench } from 'tinybench'
import { jsonrepair } from '../../lib/esm/index.js'
import { formatTaskResult } from './utils/formatTaskResult.mjs'
import { table } from './utils/table.mjs'
const text = generateText(100)
console.log(`Document size: ${Math.round(text.length / 1024)} kB`)
assert.strictEqual(text, jsonrepair(text))
const bench = new Bench().add('jsonrepair', () => jsonrepair(text))
await bench.run()
table(bench.tasks.map(formatTaskResult))
/**
* create a JSON document containing all different things that JSON can have:
* - nested objects and arrays
* - strings (with control chars and unicode)
* - numbers (various notations)
* - boolean
* - null
* - indentation and newlines
*/
function generateText(itemCount = 100) {
const json = [...new Array(itemCount)].map((_value, index) => {
return {
id: index,
name: `Item ${index}`,
details: {
description: 'Here we try out control characters and unicode',
newline: 'Some text with a newline \n',
tab: 'Some text with a tab \t',
unicode: 'Test with unicode characters 😀,💩',
'escaped double quote': '"abc"',
'unicode double quote': '\u0022abc\u0022'
},
isTrue: true,
isFalse: false,
isNull: null,
values: [1, 2.44481, 23.33e4, -5.71, 500023105]
}
})
return JSON.stringify(json, null, 2)
}
```
## /tools/benchmark/utils/formatTaskResult.mjs
```mjs path="/tools/benchmark/utils/formatTaskResult.mjs"
/**
* Format a result like:
*
* { name: "Task name", mean: "2.30 µs", variance: "±0.79%"}
*
* @param {import('tinybench').Task} task
* @return {{ name: string, mean: string, variance: string }}
*/
export function formatTaskResult(task) {
const name = task.name
const { variance, mean } = task.result.latency
return {
name,
mean: `${(mean * 1000).toFixed(2)} \u00b5s`,
variance: `±${((variance / mean) * 100).toFixed(2)}%`
}
}
```
## /tools/benchmark/utils/table.mjs
```mjs path="/tools/benchmark/utils/table.mjs"
export function table(items, gap = 2) {
const widths = getWidths(items)
const keys = Array.from(widths.keys())
console.log(keys.map((key) => padRight(key, widths.get(key) + gap)).join(''))
for (const item of items) {
console.log(keys.map((key) => padRight(item[key], widths.get(key) + gap)).join(''))
}
}
/**
* @param {Record<string, unknown>[]} items
* @returns {Map<any, any>}
*/
function getWidths(items) {
const widths = new Map()
for (const item of items) {
for (const key of Object.keys(item)) {
const length = item[key].length
const maxLength = widths.get(key)?.length ?? 0
const keyLength = key.length
widths.set(key, Math.max(length, maxLength, keyLength))
}
}
return widths
}
function padRight(text, len, char = ' ') {
const add = Math.max(len - text.length, 0)
return text + char.repeat(add)
}
```
## /tools/cjs/package.json
```json path="/tools/cjs/package.json"
{
"type": "commonjs"
}
```
## /tsconfig-types.json
```json path="/tsconfig-types.json"
{
"extends": "./tsconfig.json",
"compilerOptions": {
"emitDeclarationOnly": true
},
"exclude": ["src/**/*.test.ts"]
}
```
## /tsconfig.json
```json path="/tsconfig.json"
{
"compilerOptions": {
"moduleResolution": "node",
"module": "es2020",
"lib": ["es2020", "dom"],
"target": "es2020",
"sourceMap": true,
"esModuleInterop": true,
"allowJs": false,
"checkJs": false,
"noImplicitAny": true,
"declaration": true,
"declarationDir": "lib/types",
"declarationMap": true
},
"include": ["src/**/*.d.ts", "src/**/*.js", "src/**/*.ts"]
}
```
The better and more specific the context, the better the LLM can follow instructions. If the context seems verbose, the user can refine the filter using uithub. Thank you for using https://uithub.com - Perfect LLM context for any GitHub repo.