From 0fef8ae02547c85cab50d4a4b49089253668e2b9 Mon Sep 17 00:00:00 2001 From: swim Date: Fri, 24 Mar 2017 07:23:05 +1000 Subject: [PATCH 1/2] Added gulp file with required deps. --- Gulpfile.js | 175 +++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 115 +++++++++++++++++++++++++-------- 2 files changed, 265 insertions(+), 25 deletions(-) create mode 100755 Gulpfile.js diff --git a/Gulpfile.js b/Gulpfile.js new file mode 100755 index 0000000..11b3664 --- /dev/null +++ b/Gulpfile.js @@ -0,0 +1,175 @@ +const gulp = require('gulp'); +const mocha = require('gulp-mocha'); +const jshint = require('gulp-jshint'); +const watch = require('gulp-watch'); +const clean = require('gulp-clean'); +const exec = require('child_process').exec; + +const testServer = require('./test/setupServer'); +const MOCHA_TIMEOUT_S = 10, + MOCHA_TIMEOUT_MS = MOCHA_TIMEOUT_S * 1000, + COVERAGE_THRESHOLD = 95; + +let server = false; +let mochaOptions = { + reporter: 'spec', + timeout: MOCHA_TIMEOUT_MS +}; +let tests = [ + 'serve', + 'testAbstract', + 'testStatic', + 'testDynamic', + 'testPromise', + 'testRouter', + 'testScraperError', + 'testCommandLine' +]; + +/** + * @task + * start server + */ +gulp.task('serve', () => + server = testServer(gulp) +); + +/** + * @task + * stop server + */ +gulp.task('unserve', tests, () => + server.close() +); + +/** + * @task + * run all tests + */ +gulp.task('test', ['unserve']); + +/** + * @task + * watch all + */ +gulp.task('watch-all', ['serve', 'watch']); + +/** + * @task + * run watcher + */ +gulp.task('watch', () => + gulp.watch(['src/**/*.js', 'test/**/*.js', 'Gruntfile.js'], ['jshint']) +); + +/** + * @task + * run coverge + */ +gulp.task('coverage', () => + exec('istanbul cover ./node_modules/mocha/bin/_mocha -x src/PhantomWrapper.js -- -t ' + MOCHA_TIMEOUT_MS + ' --root src/ test/') +); + +/** + * @task + * run coveralls + */ +gulp.task('coveralls', () => + gulp.src('istanbul cover ./node_modules/mocha/bin/_mocha -x src/PhantomWrapper.js --report lcovonly -- -t ' + MOCHA_TIMEOUT_MS + ' -x src/PhantomWrapper.js --root src/ test/ && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js') +); + +/** + * @task + * check coverage + */ +gulp.task('check-coverage', () => + gulp.src('istanbul check-coverage --lines ' + COVERAGE_THRESHOLD + ' --statements ' + COVERAGE_THRESHOLD + ' --functions ' + COVERAGE_THRESHOLD + ' --branches ' + COVERAGE_THRESHOLD + ' ./coverage/coverage.json') +); + +/** + * @task + * run linting & unit tests + */ +gulp.task('unit', ['jshint', 'test']); + +/** + * @task + * run linting + */ +gulp.task('jshint', () => + gulp.src(['src/**/*.js', 'test/**/*.js']) + .pipe(jshint()) + .pipe(jshint.reporter('default')) +); + +/** + * @task + * run cleaning + */ +gulp.task('clean', () => + gulp.src('coverage/', {read: false}) + .pipe(clean()) +); + +/** + * @task + * Abstract test + */ +gulp.task('testAbstract', ['serve'], () => + gulp.src('test/AbstractScraper.js', {read: false}) + .pipe(mocha(mochaOptions)) +); + +/** + * @task + * Static test + */ +gulp.task('testStatic', ['serve'], () => + gulp.src('test/StaticScraper.js', {read: false}) + .pipe(mocha(mochaOptions)) +); + +/** + * @task + * Dynamic test + */ +gulp.task('testDynamic', ['serve'], () => + gulp.src('test/DynamicScraper.js', {read: false}) + .pipe(mocha(mochaOptions)) +); + +/** + * @task + * Promise test + */ +gulp.task('testPromise', ['serve'], () => + gulp.src('test/ScraperPromise.js', {read: false}) + .pipe(mocha(mochaOptions)) +); + +/** + * @task + * Router test + */ +gulp.task('testRouter', ['serve'], () => + gulp.src('test/Router.js', {read: false}) + .pipe(mocha(mochaOptions)) +); + +/** + * @task + * Error test + */ +gulp.task('testScraperError', ['serve'], () => + gulp.src('test/ScraperError.js', {read: false}) + .pipe(mocha(mochaOptions)) +); + +/** + * @task + * CLI test + */ +gulp.task('testCommandLine', ['serve'], () => + gulp.src('test/commandLine.js', {read: false}) + .pipe(mocha(mochaOptions)) +); diff --git a/package.json b/package.json index 3016dbe..c093255 100644 --- a/package.json +++ b/package.json @@ -1,40 +1,66 @@ { - "name": "scraperjs", - "version": "1.2.0", - "description": "A complete and versatile web scraper.", - "main": "./src/Scraper.js", - "keywords": [ - "scraper", - "scraping", - "web" + "_args": [ + [ + { + "raw": "scraperjs", + "scope": null, + "escapedName": "scraperjs", + "name": "scraperjs", + "rawSpec": "", + "spec": "latest", + "type": "tag" + }, + "C:\\cygwin64\\home\\2527\\phantomjs\\re" + ] ], - "bin": "./bin/scraperjs", - "scripts": { - "test": "grunt test" + "_from": "scraperjs@latest", + "_id": "scraperjs@1.2.0", + "_inCache": true, + "_installable": true, + "_location": "/scraperjs", + "_nodeVersion": "4.1.0", + "_npmUser": { + "name": "ruipgil", + "email": "ruiweb@gmail.com" }, - "repository": { - "type": "git", - "url": "git://github.com/ruipgil/scraperjs.git" + "_npmVersion": "2.14.3", + "_phantomChildren": {}, + "_requested": { + "raw": "scraperjs", + "scope": null, + "escapedName": "scraperjs", + "name": "scraperjs", + "rawSpec": "", + "spec": "latest", + "type": "tag" + }, + "_requiredBy": [ + "#USER", + "/" + ], + "_resolved": "https://registry.npmjs.org/scraperjs/-/scraperjs-1.2.0.tgz", + "_shasum": "93dd91b80f27cd542fdc513a6c5deace1e6ceef3", + "_shrinkwrap": null, + "_spec": "scraperjs", + "_where": "C:\\cygwin64\\home\\2527\\phantomjs\\re", + "author": { + "name": "Rui Gil" + }, + "bin": { + "scraperjs": "./bin/scraperjs" }, "bugs": { "url": "https://github.com/ruipgil/scraperjs/issues" }, - "gitHead": "c58be022438e49564597bbb3ad7c036d610744f8", - "homepage": "https://github.com/ruipgil/scraperjs", - "author": "Rui Gil", - "license": "MIT", "dependencies": { "async": "^1.5.0", "cheerio": "^0.19.0", + "commander": "^2.9.0", "jquery": "^2.1.4", "phantom": "^0.8.4", - "request": "^2.67.0", - "commander": "^2.9.0" - }, - "readmeFilename": "README.md", - "directories": { - "test": "test" + "request": "^2.67.0" }, + "description": "A complete and versatile web scraper.", "devDependencies": { "coveralls": "^2.11.1", "express": "^4.8.3", @@ -45,10 +71,49 @@ "grunt-contrib-watch": "^0.6.1", "grunt-exec": "^0.4.6", "grunt-mocha-test": "^0.11.0", + "gulp": "^3.9.1", + "gulp-clean": "^0.3.2", + "gulp-jshint": "^2.0.4", + "gulp-mocha": "^4.1.0", + "gulp-watch": "^4.3.11", "istanbul": "^0.3.0", + "jshint": "^2.9.4", "mocha": "^1.21.4" }, + "directories": { + "test": "test" + }, + "dist": { + "shasum": "93dd91b80f27cd542fdc513a6c5deace1e6ceef3", + "tarball": "https://registry.npmjs.org/scraperjs/-/scraperjs-1.2.0.tgz" + }, "engines": { "node": ">=0.10" - } + }, + "gitHead": "c58be022438e49564597bbb3ad7c036d610744f8", + "homepage": "https://github.com/ruipgil/scraperjs", + "keywords": [ + "scraper", + "scraping", + "web" + ], + "license": "MIT", + "main": "./src/Scraper.js", + "maintainers": [ + { + "name": "ruipgil", + "email": "ruiweb@gmail.com" + } + ], + "name": "scraperjs", + "optionalDependencies": {}, + "readme": "ERROR: No README data found!", + "repository": { + "type": "git", + "url": "git://github.com/ruipgil/scraperjs.git" + }, + "scripts": { + "test": "grunt test" + }, + "version": "1.2.0" } From 530b3012f030bfabeaa84b8ef102d9ee738d7767 Mon Sep 17 00:00:00 2001 From: swim Date: Fri, 24 Mar 2017 07:33:32 +1000 Subject: [PATCH 2/2] Clean package.json file. --- package.json | 111 ++++++++++++--------------------------------------- 1 file changed, 26 insertions(+), 85 deletions(-) diff --git a/package.json b/package.json index c093255..dd12fee 100644 --- a/package.json +++ b/package.json @@ -1,66 +1,40 @@ { - "_args": [ - [ - { - "raw": "scraperjs", - "scope": null, - "escapedName": "scraperjs", - "name": "scraperjs", - "rawSpec": "", - "spec": "latest", - "type": "tag" - }, - "C:\\cygwin64\\home\\2527\\phantomjs\\re" - ] - ], - "_from": "scraperjs@latest", - "_id": "scraperjs@1.2.0", - "_inCache": true, - "_installable": true, - "_location": "/scraperjs", - "_nodeVersion": "4.1.0", - "_npmUser": { - "name": "ruipgil", - "email": "ruiweb@gmail.com" - }, - "_npmVersion": "2.14.3", - "_phantomChildren": {}, - "_requested": { - "raw": "scraperjs", - "scope": null, - "escapedName": "scraperjs", - "name": "scraperjs", - "rawSpec": "", - "spec": "latest", - "type": "tag" - }, - "_requiredBy": [ - "#USER", - "/" + "name": "scraperjs", + "version": "1.2.0", + "description": "A complete and versatile web scraper.", + "main": "./src/Scraper.js", + "keywords": [ + "scraper", + "scraping", + "web" ], - "_resolved": "https://registry.npmjs.org/scraperjs/-/scraperjs-1.2.0.tgz", - "_shasum": "93dd91b80f27cd542fdc513a6c5deace1e6ceef3", - "_shrinkwrap": null, - "_spec": "scraperjs", - "_where": "C:\\cygwin64\\home\\2527\\phantomjs\\re", - "author": { - "name": "Rui Gil" + "bin": "./bin/scraperjs", + "scripts": { + "test": "grunt test" }, - "bin": { - "scraperjs": "./bin/scraperjs" + "repository": { + "type": "git", + "url": "git://github.com/ruipgil/scraperjs.git" }, "bugs": { "url": "https://github.com/ruipgil/scraperjs/issues" }, + "gitHead": "c58be022438e49564597bbb3ad7c036d610744f8", + "homepage": "https://github.com/ruipgil/scraperjs", + "author": "Rui Gil", + "license": "MIT", "dependencies": { "async": "^1.5.0", "cheerio": "^0.19.0", - "commander": "^2.9.0", "jquery": "^2.1.4", "phantom": "^0.8.4", - "request": "^2.67.0" + "request": "^2.67.0", + "commander": "^2.9.0" + }, + "readmeFilename": "README.md", + "directories": { + "test": "test" }, - "description": "A complete and versatile web scraper.", "devDependencies": { "coveralls": "^2.11.1", "express": "^4.8.3", @@ -80,40 +54,7 @@ "jshint": "^2.9.4", "mocha": "^1.21.4" }, - "directories": { - "test": "test" - }, - "dist": { - "shasum": "93dd91b80f27cd542fdc513a6c5deace1e6ceef3", - "tarball": "https://registry.npmjs.org/scraperjs/-/scraperjs-1.2.0.tgz" - }, "engines": { "node": ">=0.10" - }, - "gitHead": "c58be022438e49564597bbb3ad7c036d610744f8", - "homepage": "https://github.com/ruipgil/scraperjs", - "keywords": [ - "scraper", - "scraping", - "web" - ], - "license": "MIT", - "main": "./src/Scraper.js", - "maintainers": [ - { - "name": "ruipgil", - "email": "ruiweb@gmail.com" - } - ], - "name": "scraperjs", - "optionalDependencies": {}, - "readme": "ERROR: No README data found!", - "repository": { - "type": "git", - "url": "git://github.com/ruipgil/scraperjs.git" - }, - "scripts": { - "test": "grunt test" - }, - "version": "1.2.0" -} + } +} \ No newline at end of file