From da8f7830cadb608e0b1c265cdea77cfadd151822 Mon Sep 17 00:00:00 2001 From: Girish Gopaul Date: Thu, 11 Jan 2018 13:56:06 +0400 Subject: [PATCH 1/2] Added the uploadBoxesCsv method and changed the command line tool to upload CSV file --- bin/sharinpix.coffee | 15 +++++ package-lock.json | 44 +++++++------ package.json | 5 +- src/sharinpix.coffee | 131 ++++++++++++++++++++++++++++++++++++++- src/utils/convert-csv.js | 54 ++++++++++++++++ 5 files changed, 226 insertions(+), 23 deletions(-) create mode 100644 src/utils/convert-csv.js diff --git a/bin/sharinpix.coffee b/bin/sharinpix.coffee index 386d738..b1e1e88 100644 --- a/bin/sharinpix.coffee +++ b/bin/sharinpix.coffee @@ -42,5 +42,20 @@ module.exports = -> Sharinpix.multiupload data else console.log 'Wrong parameters' + when 'upload_boxes_csv' + csvPath = process.argv[3] + albumId = process.argv[4] + outFile = process.argv[5] + if csvPath && albumId + Sharinpix.configure sharinpixSecretUrl + Sharinpix.upload_boxes_csv(fs.createReadStream(csvPath), albumId).then (results) -> + writeStream = fs.createWriteStream(outFile || 'sharinpix.log') + writeStream.write JSON.stringify(results) + writeStream.end() + console.log results, 'SUCCESS' + , (error) -> + console.log error, 'ERROR' + else + console.log 'Wrong parameters' else console.log 'Please use appropriate action. Available:\n upload []\n multiupload ' diff --git a/package-lock.json b/package-lock.json index b48d470..b6c9b90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,9 @@ { "name": "sharinpix", - "version": "0.0.5", + "version": "0.0.7", "lockfileVersion": 1, "requires": true, "dependencies": { - "Base64": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/Base64/-/Base64-0.2.1.tgz", - "integrity": "sha1-ujpCMHCOGGcFBl5mur3Uw1z2ACg=", - "dev": true - }, "abbrev": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.0.tgz", @@ -811,6 +805,12 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "Base64": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/Base64/-/Base64-0.2.1.tgz", + "integrity": "sha1-ujpCMHCOGGcFBl5mur3Uw1z2ACg=", + "dev": true + }, "base64-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.1.tgz", @@ -2250,8 +2250,7 @@ "graceful-fs": { "version": "4.1.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", - "dev": true + "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" }, "graceful-readlink": { "version": "1.0.1", @@ -2878,6 +2877,14 @@ "integrity": "sha1-BUNS5MTIDIbAkjh31EneF2pzLI0=", "dev": true }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "4.1.11" + } + }, "jsonify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", @@ -5280,6 +5287,14 @@ "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=", "dev": true }, + "string_decoder": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", + "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", + "requires": { + "safe-buffer": "5.1.1" + } + }, "string-extended": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/string-extended/-/string-extended-0.0.8.tgz", @@ -5302,14 +5317,6 @@ "strip-ansi": "3.0.1" } }, - "string_decoder": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", - "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", - "requires": { - "safe-buffer": "5.1.1" - } - }, "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", @@ -5711,8 +5718,7 @@ "uuid": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==", - "dev": true + "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" }, "validate-npm-package-license": { "version": "3.0.1", diff --git a/package.json b/package.json index ae742e6..985334d 100644 --- a/package.json +++ b/package.json @@ -50,8 +50,11 @@ "async": "^2.5.0", "coffee-script": "1.10.0", "fast-csv": "^2.4.1", + "jsonfile": "^4.0.0", "jsrsasign": "5.0.12", + "lodash": "^4.17.4", "promise-polyfill": "^6.1.0", - "superagent": "2.0.0" + "superagent": "2.0.0", + "uuid": "^3.1.0" } } diff --git a/src/sharinpix.coffee b/src/sharinpix.coffee index 4ce3aea..46ef196 100644 --- a/src/sharinpix.coffee +++ b/src/sharinpix.coffee @@ -1,10 +1,12 @@ -jsrsasign = require 'jsrsasign' -superagent = require 'superagent' url = require 'url' +_ = require 'lodash' path = require 'path' async = require 'async' fastCsv = require 'fast-csv' -Promise = require('promise-polyfill') +Promise = require 'promise-polyfill' +jsrsasign = require 'jsrsasign' +superagent = require 'superagent' +convertCsv = require './utils/convert-csv' class Sharinpix constructor: (@options)-> @@ -109,6 +111,127 @@ class Sharinpix ) .on 'end', -> async.parallelLimit uploads, 2, multiupload_callback + uploadBoxesCsv: (bufferStream, albumId) -> + new Promise (resolve, reject) => + parallelRequests = (tasks, limit, callback) -> + async.parallelLimit tasks, limit, (err, results) -> + callback err, results + return + return + convertCsv bufferStream, (data) => + abilities = {} + # check import status + checkImports = (importResults) => + importTasks = [] + _.each importResults, (importVal) => + if importVal['value'] == undefined + return + imp = importVal.value + importTasks.push (callback) => + async.retry { + errorFilter: (err) => + err.image_id == null + interval: 3000 + times: 20 + }, ((done) => + @get('/imports/' + imp.id, admin: true).then ((impResult) => + if impResult.image_id == null + done impResult + else + done null, impResult + return + ), (impError) => + done null, {} + return + return + ), (err, result) => + # if err != undefined and err != null and err.length > 0 + # console.log '############## import status error : ' + JSON.stringify(err) + # if result != undefined and result != null and result.length > 0 + # console.log '############## import status result : ' + result + callback err, result + return + return + return + parallelRequests importTasks, 5, (errors, results) => + if results != null and results.length > 0 + createBox results + if errors != null and errors.length > 0 + console.log '### import errors: ' + errors.length + return + return + createBox = (importRes) => + boxTasks = [] + _.each importRes, (imp) => + if imp == undefined or imp == null or imp == {} or imp.image_id == null + console.log 'there was an error on imp here' + else + image = data[imp.params.metadatas.externalId] + einsteinBoxes = image.boxes + _.each einsteinBoxes, (box) => + box.image_id = imp.image_id + boxTasks.push async.reflect((callback) => + @post('/images/' + imp.image_id + '/einstein_box', box, claims).then ((res) => + callback null, res + return + ), (err) => + callback err, null + return + return + ) + return + return + setTimeout (=> + # console.log 'wait 10s' + parallelRequests boxTasks, 5, (err, result) => + if err + reject err + else + resolve result + return + return + ), 10000 + return + abilities[albumId] = Access: + see: true + image_upload: true + einstein_box: true + claims = abilities: abilities + parallelTasks = [] + # creating imports + _.each data, (item, key) => + # console.log(key); + body = + album_id: albumId + filename: item.image_name + url: item.image_url + import_type: 'url' + metadatas: externalId: key + parallelTasks.push async.reflect((callback) => + @post('/imports', body, claims).then ((res) => + if res == null + # console.log JSON.stringify(body) + callback body, null + else + callback null, + id: res.id + external_id: key + return + ), (err) => + callback body, null + return + return + ) + return + parallelRequests parallelTasks, 5, (errors, results) => + if results != null and results.length > 0 + console.log '@@@@ import success: ' + JSON.stringify(results) + checkImports results + if errors != null and errors.length > 0 + console.log '@@@@ errors success: ' + errors.length + return + return + return _options = undefined Sharinpix.configure = (options)-> @@ -143,5 +266,7 @@ Sharinpix.multiupload = -> Sharinpix.get_instance().multiupload arguments... Sharinpix.image_delete = -> Sharinpix.get_instance().image_delete arguments... +Sharinpix.upload_boxes_csv = -> + Sharinpix.get_instance().uploadBoxesCsv arguments... module.exports = Sharinpix diff --git a/src/utils/convert-csv.js b/src/utils/convert-csv.js new file mode 100644 index 0000000..95fa8fe --- /dev/null +++ b/src/utils/convert-csv.js @@ -0,0 +1,54 @@ +const fs = require('fs'); +const csv = require('fast-csv'); +const uuidV4 = require('uuid/v4'); +const jsonfile = require('jsonfile'); + +module.exports = function(inputFile, callback) { + var stream = typeof inputFile === 'string' ? fs.createReadStream(inputFile) : inputFile; + // var stream = fs.createReadStream(inputFile); + var header = true; + var output = {}; + + var csvStream = csv({quote: '"'}) + .on("data", function(data){ + if (!header){ + var externalId = uuidV4(); + var image_name = data[0]; + var image_url = data[1]; + var image_width = data[2]; + var image_height = data[3]; + var boxes = []; + for(var i = 4; i < data.length; i++){ + if (data[i] !== null && data[i] !== undefined && data[i] != ''){ + boxes.push( + convertBox(JSON.parse(data[i]), image_width, image_height) + ); + } + + } + + output[externalId] = { + image_name: image_name, + image_url: image_url, + image_width: image_width, + image_height: image_height, + boxes: boxes + } + } + header = false; + }) + .on("end", function(){ + jsonfile.writeFileSync('./output.json', output); + callback(output); + }); + + function convertBox(box, image_width, image_height) { + let percentageWidth = (box.width / image_width) * 100; + let percentageHeight = (box.height / image_height) * 100; + let percentageX = (box.x / image_width) * 100; + let percentageY = (box.y / image_height) * 100; + return {label: box.label, width: percentageWidth, height: percentageHeight, left: percentageX, top: percentageY }; + } + + stream.pipe(csvStream); +}; \ No newline at end of file From e02e6c97e25743c3c5f3087b92e78dcb96d06a0e Mon Sep 17 00:00:00 2001 From: Girish Gopaul Date: Thu, 11 Jan 2018 13:58:19 +0400 Subject: [PATCH 2/2] Added the uploadBoxesCsv method and changed the command line tool to upload CSV file --- bin/sharinpix.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/sharinpix.coffee b/bin/sharinpix.coffee index b1e1e88..7492f1e 100644 --- a/bin/sharinpix.coffee +++ b/bin/sharinpix.coffee @@ -58,4 +58,4 @@ module.exports = -> else console.log 'Wrong parameters' else - console.log 'Please use appropriate action. Available:\n upload []\n multiupload ' + console.log 'Please use appropriate action. Available:\n upload []\n multiupload \n upload_boxes_csv []'