Csv box import

bin/sharinpix.coffee

@@ -42,5 +42,20 @@ module.exports = ->
           Sharinpix.multiupload data
       else
         console.log 'Wrong parameters'
+    when 'upload_boxes_csv'
+      csvPath = process.argv[3]
+      albumId = process.argv[4]
+      outFile = process.argv[5]
+      if csvPath && albumId
+        Sharinpix.configure sharinpixSecretUrl
+        Sharinpix.upload_boxes_csv(fs.createReadStream(csvPath), albumId).then (results) ->
+          writeStream = fs.createWriteStream(outFile || 'sharinpix.log')
+          writeStream.write JSON.stringify(results)
+          writeStream.end()
+          console.log results, 'SUCCESS'
+        , (error) ->
+          console.log error, 'ERROR'
+      else
+        console.log 'Wrong parameters'
     else
-      console.log 'Please use appropriate action. Available:\n  upload <image path> <album id> [<JSON metadatas>]\n  multiupload <csv file>'
+      console.log 'Please use appropriate action. Available:\n  upload <image path> <album id> [<JSON metadatas>]\n  multiupload <csv file>\n  upload_boxes_csv <csv file> <album id> [<output log file>]'

package.json

@@ -50,8 +50,11 @@
     "async": "^2.5.0",
     "coffee-script": "1.10.0",
     "fast-csv": "^2.4.1",
+    "jsonfile": "^4.0.0",
     "jsrsasign": "5.0.12",
+    "lodash": "^4.17.4",
     "promise-polyfill": "^6.1.0",
-    "superagent": "2.0.0"
+    "superagent": "2.0.0",
+    "uuid": "^3.1.0"
   }
 }

src/sharinpix.coffee

@@ -1,10 +1,12 @@
-jsrsasign = require 'jsrsasign'
-superagent = require 'superagent'
 url = require 'url'
+_ = require 'lodash'
 path = require 'path'
 async = require 'async'
 fastCsv = require 'fast-csv'
-Promise = require('promise-polyfill')
+Promise = require 'promise-polyfill'
+jsrsasign = require 'jsrsasign'
+superagent = require 'superagent'
+convertCsv = require './utils/convert-csv'
 
 class Sharinpix
   constructor: (@options)->
@@ -109,6 +111,127 @@ class Sharinpix
           )
       .on 'end', ->
         async.parallelLimit uploads, 2, multiupload_callback
+  uploadBoxesCsv: (bufferStream, albumId) ->
+    new Promise (resolve, reject) =>
+      parallelRequests = (tasks, limit, callback) ->
+        async.parallelLimit tasks, limit, (err, results) ->
+          callback err, results
+          return
+        return
+      convertCsv bufferStream, (data) =>
+        abilities = {}
+        # check import status
+        checkImports = (importResults) =>
+          importTasks = []
+          _.each importResults, (importVal) =>
+            if importVal['value'] == undefined
+              return
+            imp = importVal.value
+            importTasks.push (callback) =>
+              async.retry {
+                errorFilter: (err) =>
+                  err.image_id == null
+                interval: 3000
+                times: 20
+              }, ((done) =>
+                @get('/imports/' + imp.id, admin: true).then ((impResult) =>
+                  if impResult.image_id == null
+                    done impResult
+                  else
+                    done null, impResult
+                  return
+                ), (impError) =>
+                  done null, {}
+                  return
+                return
+              ), (err, result) =>
+                # if err != undefined and err != null and err.length > 0
+                #   console.log '############## import status error : ' + JSON.stringify(err)
+                # if result != undefined and result != null and result.length > 0
+                #   console.log '############## import status result : ' + result
+                callback err, result
+                return
+              return
+            return
+          parallelRequests importTasks, 5, (errors, results) =>
+            if results != null and results.length > 0
+              createBox results
+            if errors != null and errors.length > 0
+              console.log '### import errors: ' + errors.length
+            return
+          return
+        createBox = (importRes) =>
+          boxTasks = []
+          _.each importRes, (imp) =>
+            if imp == undefined or imp == null or imp == {} or imp.image_id == null
+              console.log 'there was an error on imp here'
+            else
+              image = data[imp.params.metadatas.externalId]
+              einsteinBoxes = image.boxes
+              _.each einsteinBoxes, (box) =>
+                box.image_id = imp.image_id
+                boxTasks.push async.reflect((callback) =>
+                  @post('/images/' + imp.image_id + '/einstein_box', box, claims).then ((res) =>
+                    callback null, res
+                    return
+                  ), (err) =>
+                    callback err, null
+                    return
+                  return
+                )
+                return
+            return
+          setTimeout (=>
+            # console.log 'wait 10s'
+            parallelRequests boxTasks, 5, (err, result) =>
+              if err
+                reject err
+              else
+                resolve result
+              return
+            return
+          ), 10000
+          return
+        abilities[albumId] = Access:
+          see: true
+          image_upload: true
+          einstein_box: true
+        claims = abilities: abilities
+        parallelTasks = []
+        # creating imports
+        _.each data, (item, key) =>
+          # console.log(key);
+          body = 
+            album_id: albumId
+            filename: item.image_name
+            url: item.image_url
+            import_type: 'url'
+            metadatas: externalId: key
+          parallelTasks.push async.reflect((callback) =>
+            @post('/imports', body, claims).then ((res) =>
+              if res == null
+                # console.log JSON.stringify(body)
+                callback body, null
+              else
+                callback null,
+                  id: res.id
+                  external_id: key
+              return
+            ), (err) =>
+              callback body, null
+              return
+            return
+          )
+          return
+        parallelRequests parallelTasks, 5, (errors, results) =>
+          if results != null and results.length > 0
+            console.log '@@@@ import success: ' + JSON.stringify(results)
+            checkImports results
+          if errors != null and errors.length > 0
+            console.log '@@@@ errors success: ' + errors.length
+          return
+        return
+      return
 
 _options = undefined
 Sharinpix.configure = (options)->
@@ -143,5 +266,7 @@ Sharinpix.multiupload = ->
   Sharinpix.get_instance().multiupload arguments...
 Sharinpix.image_delete = ->
   Sharinpix.get_instance().image_delete arguments...
+Sharinpix.upload_boxes_csv = ->
+  Sharinpix.get_instance().uploadBoxesCsv arguments...
 
 module.exports = Sharinpix

src/utils/convert-csv.js

@@ -0,0 +1,54 @@
+const fs = require('fs');
+const csv = require('fast-csv');
+const uuidV4 = require('uuid/v4');
+const jsonfile = require('jsonfile');
+
+module.exports = function(inputFile, callback) {
+    var stream = typeof inputFile === 'string' ? fs.createReadStream(inputFile) : inputFile;
+    // var stream = fs.createReadStream(inputFile);
+    var header = true;
+    var output = {};
+
+    var csvStream = csv({quote: '"'})
+        .on("data", function(data){
+            if (!header){
+                var externalId = uuidV4();
+                var image_name = data[0];
+                var image_url = data[1];
+                var image_width = data[2];
+                var image_height = data[3];
+                var boxes = [];
+                for(var i = 4; i < data.length; i++){
+                    if (data[i] !== null && data[i] !== undefined && data[i] != ''){
+                        boxes.push(
+                            convertBox(JSON.parse(data[i]), image_width, image_height)
+                        );
+                    }
+
+                }
+
+                output[externalId] = {
+                    image_name: image_name,
+                    image_url: image_url,
+                    image_width: image_width,
+                    image_height: image_height,
+                    boxes: boxes
+                }
+            }
+            header = false;
+        })
+        .on("end", function(){
+            jsonfile.writeFileSync('./output.json', output);
+            callback(output);
+        });
+
+    function convertBox(box, image_width, image_height) {
+        let percentageWidth = (box.width / image_width) * 100;
+        let percentageHeight = (box.height / image_height) * 100;
+        let percentageX = (box.x / image_width) * 100;
+        let percentageY = (box.y / image_height) * 100;
+        return {label: box.label, width: percentageWidth, height: percentageHeight, left: percentageX, top: percentageY };
+    }
+
+    stream.pipe(csvStream);
+};