From 3aa7f682b770596d345dac378c93ea0a94c1588a Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Mon, 23 Mar 2015 08:25:11 -0400 Subject: [PATCH 01/13] switch to through2 --- index.js | 48 +++++++++++++++++++++++------------------------- package.json | 3 ++- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/index.js b/index.js index 79e5efa..dbf0ec5 100755 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ #! /usr/bin/env node var Parser = require('jsonparse') - , through = require('through') + , through = require('through2').obj /* @@ -15,15 +15,11 @@ var Parser = require('jsonparse') exports.parse = function (path, map) { var parser = new Parser() - var stream = through(function (chunk) { - if('string' === typeof chunk) - chunk = new Buffer(chunk) + var stream = through(function (chunk, encoding, next) { + if('string' === typeof encoding) + chunk = new Buffer(chunk, encoding) parser.write(chunk) - }, - function (data) { - if(data) - stream.write(data) - stream.queue(null) + next() }) if('string' === typeof path) @@ -83,7 +79,7 @@ exports.parse = function (path, map) { var data = this.value[this.key] if(null != data) if(null != (data = map ? map(data, actualPath) : data)) - stream.queue(data) + stream.push(data) delete this.value[this.key] for(var k in this.stack) this.stack[k].value = null @@ -95,7 +91,7 @@ exports.parse = function (path, map) { if (this.stack.length === 0) { if (stream.root) { if(!path) - stream.queue(stream.root) + stream.push(stream.root) count = 0; stream.root = null; } @@ -143,17 +139,18 @@ exports.stringify = function (op, sep, cl, indent) { var stream , first = true , anyData = false - stream = through(function (data) { + stream = through(function (data, _, next) { anyData = true var json = JSON.stringify(data, null, indent) - if(first) { first = false ; stream.queue(op + json)} - else stream.queue(sep + json) + if(first) { first = false ; stream.push(op + json)} + else stream.push(sep + json) + next() }, - function (data) { + function (done) { if(!anyData) - stream.queue(op) - stream.queue(cl) - stream.queue(null) + stream.push(op) + stream.push(cl) + done() }) return stream @@ -177,17 +174,18 @@ exports.stringifyObject = function (op, sep, cl, indent) { var first = true , anyData = false - stream = through(function (data) { + stream = through(function (data, _, next) { anyData = true var json = JSON.stringify(data[0]) + ':' + JSON.stringify(data[1], null, indent) - if(first) { first = false ; this.queue(op + json)} - else this.queue(sep + json) + if(first) { first = false ; this.push(op + json)} + else this.push(sep + json) + next() }, - function (data) { - if(!anyData) this.queue(op) - this.queue(cl) + function (done) { + if(!anyData) this.push(op) + this.push(cl) - this.queue(null) + done() }) return stream diff --git a/package.json b/package.json index 501ef9a..18db90d 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ ], "dependencies": { "jsonparse": "~1.0.0", - "through": ">=2.2.7 <3" + "through": ">=2.2.7 <3", + "through2": "^0.6.3" }, "devDependencies": { "it-is": "~1", From 39b9f13301af3313e2e702264e8ec876ce6ace9a Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Mon, 23 Mar 2015 08:34:22 -0400 Subject: [PATCH 02/13] fixed one test that doesn't work with streams 2, and clarify multiple_objects_error test --- test/destroy_missing.js | 2 +- test/multiple_objects_error.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/destroy_missing.js b/test/destroy_missing.js index 315fdc8..8bc8544 100644 --- a/test/destroy_missing.js +++ b/test/destroy_missing.js @@ -11,7 +11,7 @@ var server = net.createServer(function(client) { console.log('close') console.error('PASSED'); server.close(); - }); + }).on('data', function (){}) client.pipe(parser); var n = 4 client.on('data', function () { diff --git a/test/multiple_objects_error.js b/test/multiple_objects_error.js index 83d113b..fb42023 100644 --- a/test/multiple_objects_error.js +++ b/test/multiple_objects_error.js @@ -12,11 +12,13 @@ var server = net.createServer(function(client) { var parser = JSONStream.parse(); parser.on('error', function(err) { console.log(err); + console.error('PASSED') server.close(); }); parser.on('end', function() { console.log('END'); + console.log('FAILED') server.close(); }); client.pipe(parser); From 5bb776e9f184de1d3769e225ff6131dab5251675 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Mon, 23 Mar 2015 08:38:32 -0400 Subject: [PATCH 03/13] rm through1 --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 18db90d..1e3129d 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ ], "dependencies": { "jsonparse": "~1.0.0", - "through": ">=2.2.7 <3", "through2": "^0.6.3" }, "devDependencies": { From dad7b880c58e372582452a12f54cd523a13ba955 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Mon, 23 Mar 2015 08:56:55 -0400 Subject: [PATCH 04/13] readable streams --- index.js | 152 +++++++++++++++++++++++++++++++-------------------- package.json | 3 +- 2 files changed, 96 insertions(+), 59 deletions(-) diff --git a/index.js b/index.js index dbf0ec5..55f8d16 100755 --- a/index.js +++ b/index.js @@ -1,7 +1,8 @@ #! /usr/bin/env node var Parser = require('jsonparse') - , through = require('through2').obj + , Transform = require('readable-stream').Transform + , inherits = require('inherits') /* @@ -12,16 +13,22 @@ var Parser = require('jsonparse') */ -exports.parse = function (path, map) { +exports.parse = Parse; - var parser = new Parser() - var stream = through(function (chunk, encoding, next) { - if('string' === typeof encoding) - chunk = new Buffer(chunk, encoding) - parser.write(chunk) - next() +inherits(Parse, Transform) + +function Parse(path, map) { + if (!(this instanceof Parse)) + return new Parse(path, map) + + Transform.call(this, { + objectMode: true }) + var stream = this; + this.root = null; + var parser = this.parser = new Parser() + if('string' === typeof path) path = path.split('.').map(function (e) { if (e === '*') @@ -105,7 +112,13 @@ exports.parse = function (path, map) { } - return stream +} + +Parse.prototype._transform = function (chunk, encoding, next) { + if('string' === typeof encoding) + chunk = new Buffer(chunk, encoding) + this.parser.write(chunk) + next() } function check (x, y) { @@ -120,77 +133,100 @@ function check (x, y) { return false } -exports.stringify = function (op, sep, cl, indent) { - indent = indent || 0 +exports.stringify = Stringify; + +inherits(Stringify, Transform) + +function Stringify(op, sep, cl, indent) { + if (!(this instanceof Stringify)) + return new Stringify(op, sep, cl, indent) + + Transform.call(this, { + objectMode: true + }) + + this.indent = indent || 0 if (op === false){ - op = '' - sep = '\n' - cl = '' + this.op = '' + this.sep = '\n' + this.cl = '' } else if (op == null) { - op = '[\n' - sep = '\n,\n' - cl = '\n]\n' + this.op = '[\n' + this.sep = '\n,\n' + this.cl = '\n]\n' } //else, what ever you like - var stream - , first = true - , anyData = false - stream = through(function (data, _, next) { - anyData = true - var json = JSON.stringify(data, null, indent) - if(first) { first = false ; stream.push(op + json)} - else stream.push(sep + json) - next() - }, - function (done) { - if(!anyData) - stream.push(op) - stream.push(cl) - done() - }) + this.first = true + this.anyData = false +} - return stream +Stringify.prototype._transform = function (data, _, next) { + this.anyData = true + var json = JSON.stringify(data, null, this.indent) + if(this.first) { + this.first = false; + this.push(this.op + json) + } + else this.push(this.sep + json) + next() +} + +Stringify.prototype._flush = function (done) { + if(!this.anyData) + this.push(this.op) + this.push(this.cl) + done() } -exports.stringifyObject = function (op, sep, cl, indent) { - indent = indent || 0 +exports.stringifyObject = StringifyObject; + +inherits(StringifyObject, Transform) + +function StringifyObject (op, sep, cl, indent) { + if (!(this instanceof StringifyObject)) + return new StringifyObject(op, sep, cl, indent) + + Transform.call(this, { + objectMode: true + }) + + this.indent = indent || 0 if (op === false){ - op = '' - sep = '\n' - cl = '' + this.op = '' + this.sep = '\n' + this.cl = '' } else if (op == null) { - op = '{\n' - sep = '\n,\n' - cl = '\n}\n' + this.op = '{\n' + this.sep = '\n,\n' + this.cl = '\n}\n' } //else, what ever you like - var first = true - , anyData = false - stream = through(function (data, _, next) { - anyData = true - var json = JSON.stringify(data[0]) + ':' + JSON.stringify(data[1], null, indent) - if(first) { first = false ; this.push(op + json)} - else this.push(sep + json) - next() - }, - function (done) { - if(!anyData) this.push(op) - this.push(cl) - - done() - }) + this.first = true + this.anyData = false +} +StringifyObject.prototype._transform = function (data, _, next) { + this.anyData = true + var json = JSON.stringify(data[0]) + ':' + JSON.stringify(data[1], null, this.indent) + if(this.first) { this.first = false ; this.push(this.op + json)} + else this.push(this.sep + json) + next() +} +StringifyObject.prototype._flush = function (done) { + if(!this.anyData) this.push(this.op) + this.push(this.cl) - return stream + done() } + if(!module.parent && process.title !== 'browser') { process.stdin .pipe(exports.parse(process.argv[2])) diff --git a/package.json b/package.json index 1e3129d..a278824 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,9 @@ "parsing" ], "dependencies": { + "inherits": "^2.0.1", "jsonparse": "~1.0.0", - "through2": "^0.6.3" + "readable-stream": "^1.0.33" }, "devDependencies": { "it-is": "~1", From a76d6e95cb0d6820aa873a33978f401998c075ea Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 1 May 2015 08:03:36 -0400 Subject: [PATCH 05/13] name change --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a278824..fbeb728 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { - "name": "JSONStream", + "name": "JSONStream2", "version": "1.0.1", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", - "homepage": "http://github.com/dominictarr/JSONStream", + "homepage": "http://github.com/calvinmetcalf/JSONStream", "repository": { "type": "git", - "url": "git://github.com/dominictarr/JSONStream.git" + "url": "git://github.com/calvinmetcalf/JSONStream.git" }, "keywords": [ "json", From 2d8228f04e7c800e7b613fe9d0f8cc893ff87b63 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 1 May 2015 08:03:41 -0400 Subject: [PATCH 06/13] 2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fbeb728..ed3cc77 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "JSONStream2", - "version": "1.0.1", + "version": "2.0.0", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "homepage": "http://github.com/calvinmetcalf/JSONStream", "repository": { From 1cc76738820e87a64759614a1dd23779361bee34 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 1 May 2015 08:04:14 -0400 Subject: [PATCH 07/13] lowercase --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ed3cc77..e8eca34 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "JSONStream2", + "name": "jsonstream2", "version": "2.0.0", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "homepage": "http://github.com/calvinmetcalf/JSONStream", From 1b7f31fa21a3951de740952edb0196af1124e1c8 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 1 May 2015 08:04:17 -0400 Subject: [PATCH 08/13] 2.0.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e8eca34..ae17cd0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jsonstream2", - "version": "2.0.0", + "version": "2.0.1", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "homepage": "http://github.com/calvinmetcalf/JSONStream", "repository": { From 6b02982371661f19354c2eea2c2584e08d7655e8 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Mon, 11 May 2015 16:03:32 -0400 Subject: [PATCH 09/13] modify title --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ae17cd0..8a2e62c 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "jsonstream2", + "name": "jsonstream2a", "version": "2.0.1", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "homepage": "http://github.com/calvinmetcalf/JSONStream", From aaa3f1146d7609fc4829367b5e4966dca59f1ca5 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 22 May 2015 15:49:08 -0400 Subject: [PATCH 10/13] fix custom stringify bug --- index.js | 17 +++++++++++------ test/stringify-custom.js | 41 ++++++++++++++++++++++++++++++++++++++++ test/stringify.js | 6 +++--- 3 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 test/stringify-custom.js diff --git a/index.js b/index.js index 55f8d16..db982a2 100755 --- a/index.js +++ b/index.js @@ -156,6 +156,10 @@ function Stringify(op, sep, cl, indent) { this.sep = '\n,\n' this.cl = '\n]\n' + } else { + this.op = op + this.sep = sep + this.cl = cl } //else, what ever you like @@ -169,9 +173,11 @@ Stringify.prototype._transform = function (data, _, next) { var json = JSON.stringify(data, null, this.indent) if(this.first) { this.first = false; - this.push(this.op + json) + this.push(this.op) } - else this.push(this.sep + json) + else this.push(this.sep) + + this.push(json) next() } @@ -207,16 +213,15 @@ function StringifyObject (op, sep, cl, indent) { } - //else, what ever you like - this.first = true this.anyData = false } StringifyObject.prototype._transform = function (data, _, next) { this.anyData = true var json = JSON.stringify(data[0]) + ':' + JSON.stringify(data[1], null, this.indent) - if(this.first) { this.first = false ; this.push(this.op + json)} - else this.push(this.sep + json) + if(this.first) { this.first = false ; this.push(this.op)} + else this.push(this.sep) + this.push(json) next() } StringifyObject.prototype._flush = function (done) { diff --git a/test/stringify-custom.js b/test/stringify-custom.js new file mode 100644 index 0000000..f866a18 --- /dev/null +++ b/test/stringify-custom.js @@ -0,0 +1,41 @@ + +var fs = require ('fs') + , join = require('path').join + , file = join(__dirname, 'fixtures','all_npm.json') + , JSONStream = require('../') + , it = require('it-is').style('colour') + + function randomObj () { + return ( + Math.random () < 0.4 + ? {hello: 'eonuhckmqjk', + whatever: 236515, + lies: true, + nothing: [null], + stuff: [Math.random(),Math.random(),Math.random()] + } + : ['AOREC', 'reoubaor', {ouec: 62642}, [[[], {}, 53]]] + ) + } + +var expected = [] + , stringify = JSONStream.stringify('{"features": [', ',', ']}') + , es = require('event-stream') + , stringified = '' + , called = 0 + , count = 10 + , ended = false + +while (count --) + expected.push(randomObj()) + + es.connect( + es.readArray(expected), + stringify, + es.writeArray(function (err, lines) { + it(JSON.parse(lines.join(''))).deepEqual({ + features: expected + }) + console.error('PASSED') + }) + ) diff --git a/test/stringify.js b/test/stringify.js index b6de85e..20b9969 100644 --- a/test/stringify.js +++ b/test/stringify.js @@ -13,7 +13,7 @@ var fs = require ('fs') lies: true, nothing: [null], stuff: [Math.random(),Math.random(),Math.random()] - } + } : ['AOREC', 'reoubaor', {ouec: 62642}, [[[], {}, 53]]] ) } @@ -25,7 +25,7 @@ var expected = [] , called = 0 , count = 10 , ended = false - + while (count --) expected.push(randomObj()) @@ -34,7 +34,7 @@ while (count --) stringify, //JSONStream.parse([/./]), es.writeArray(function (err, lines) { - + it(JSON.parse(lines.join(''))).deepEqual(expected) console.error('PASSED') }) From 19c4ce43e24f635f23f890f3c54d04d53cf517a2 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 22 May 2015 15:49:15 -0400 Subject: [PATCH 11/13] 2.0.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8a2e62c..3b48b8c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jsonstream2a", - "version": "2.0.1", + "version": "2.0.2", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "homepage": "http://github.com/calvinmetcalf/JSONStream", "repository": { From 192ec9a4225c561d8539bef45e42492cf31cd2e4 Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 22 May 2015 15:50:30 -0400 Subject: [PATCH 12/13] use io-stream --- index.js | 2 +- package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index db982a2..e06e895 100755 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ #! /usr/bin/env node var Parser = require('jsonparse') - , Transform = require('readable-stream').Transform + , Transform = require('io-stream').Transform , inherits = require('inherits') /* diff --git a/package.json b/package.json index 3b48b8c..64050ce 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,8 @@ ], "dependencies": { "inherits": "^2.0.1", - "jsonparse": "~1.0.0", - "readable-stream": "^1.0.33" + "io-stream": "^2.0.2", + "jsonparse": "~1.0.0" }, "devDependencies": { "it-is": "~1", From 579466df8a57fa45cf1d0366f7c6bce2924dc3ef Mon Sep 17 00:00:00 2001 From: Calvin Metcalf Date: Fri, 22 May 2015 15:50:34 -0400 Subject: [PATCH 13/13] 2.0.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 64050ce..3ca1790 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jsonstream2a", - "version": "2.0.2", + "version": "2.0.3", "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "homepage": "http://github.com/calvinmetcalf/JSONStream", "repository": {