Skip to content

Dropping plain HTTP and handling large-size uploads #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stuart-macleod-akamai wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Dropping plain HTTP and handling large-size uploads #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 70 additions & 56 deletions lib/api-request.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,102 +2,116 @@
const Auth = require('./api-auth')
const Parser = require('./api-request-parser')
const path = require('path')
const http = require('http')
const https = require('https')
const fs = require('fs')
const HttpsProxyAgent = require('https-proxy-agent')
const { HttpsProxyAgent } = require('https-proxy-agent')

class Requestor {
constructor(requestorOptions) {
this.requestorOptions = requestorOptions
if (!this.requestorOptions instanceof Object) {
throw new TypeError('[Requestor Error] options should be an object')
} else if (!(this.requestorOptions.hostname && this.requestorOptions.keyName && this.requestorOptions.key && this.requestorOptions.ssl != undefined)) {
throw new Error('[Requestor Error] options object should contain key, keyName, hostname, and ssl attributes')
} else if (!(this.requestorOptions.hostname && this.requestorOptions.keyName && this.requestorOptions.key)) {
throw new Error('[Requestor Error] options object should contain key, keyName, and hostname attributes')
}

this.auth = new Auth({ key: this.requestorOptions.key, keyName: this.requestorOptions.keyName })
this.parser = new Parser()
if(requestorOptions.proxy) this.agent = new HttpsProxyAgent(requestorOptions.proxy);
if (requestorOptions.proxy) this.agent = new HttpsProxyAgent(requestorOptions.proxy);
}

makeRequest(requestArgs, callback) {

async makeRequest(requestArgs, callback) {
const acs_action = `version=1&action=${requestArgs.action}`
const netstoragePath = this.validatePath(requestArgs.path)
const authData = this.auth.auth(netstoragePath, acs_action)

var url = "https://" + this.requestorOptions.hostname + netstoragePath;
var options = {
method: requestArgs.method,
host: this.requestorOptions.hostname,
path: netstoragePath,
headers: {
'X-Akamai-ACS-Action': acs_action,
'X-Akamai-ACS-Auth-Data': authData.acs_auth_data,
'X-Akamai-ACS-Auth-Sign': authData.acs_auth_sign,
'Accept-Encoding': 'identity',
'User-Agent': 'NetStorageKit-Node'
'User-Agent': 'NetStorageKit-Node',
'Accept': '*/*'
}
}

if(this.agent) options.agent = this.agent;

var request = (this.requestorOptions.ssl ? https:http).request(options, (res) => {
if (this.agent) options.agent = this.agent;

// Create read stream
if (requestArgs.action == 'upload') {
var fileStats = fs.statSync(requestArgs.source);
var fileSize = fileStats.size;
var fileStream = fs.createReadStream(requestArgs.source, {
highWaterMark: 64 * 1024 // 64KB chunks by default
});
options.headers['Content-Type'] = 'application/octet-stream';
options.headers['Content-Length'] = fileSize;
}


var request = https.request(url, options, (res) => {
var rawData = ''
res.setEncoding('binary')
res
.on('data', (data) => {
rawData += data
})
.on('end', () => {
if (requestArgs.action == 'download') {
var local_destination = requestArgs.destination
if (requestArgs.path.endsWith('/')) {
callback(new Error('[Netstorage Error] Nestorage Path should be a file, not directory'), null, null)
return
} else if (local_destination == '') {
local_destination = path.basename(requestArgs.path)
} else {
.on('data', (data) => {
rawData += data
})
.on('end', () => {
if (requestArgs.action == 'download') {
var local_destination = requestArgs.destination
if (requestArgs.path.endsWith('/')) {
callback(new Error('[Netstorage Error] Nestorage Path should be a file, not directory'), null, null)
return
} else if (local_destination == '') {
local_destination = path.basename(requestArgs.path)
} else {
try {
if (fs.statSync(local_destination).isDirectory()) {
local_destination = path.join(local_destination, path.basename(requestArgs.path))
}
} catch (e) { }
}
try {
if (fs.statSync(local_destination).isDirectory()) {
local_destination = path.join(local_destination, path.basename(requestArgs.path))
fs.writeFileSync(local_destination, rawData, 'binary')
} catch (e) {
callback(e)
return
}
callback(null, res, { message: 'Download Done.' })
} else {
this.parser.parse(rawData, (err, json) => {
if (requestArgs.action == 'upload' && !rawData && res.statusCode == 200) {
// For Object Store upload response: {}
callback(null, res, { message: 'Request Processed.' })
} else {
callback(null, res, json)
}
} catch (e) {}
})
}
try {
fs.writeFileSync(local_destination, rawData, 'binary')
} catch (e) {
callback(e)
return
}
callback(null, res, {message: 'Download Done.'})
} else {
this.parser.parse(rawData, (err, json) => {
if (requestArgs.action == 'upload' && !rawData && res.statusCode == 200 ) {
// For Object Store upload response: {}
callback(null, res, {message: 'Request Processed.'})
} else {
callback(null, res, json)
}
})
}
})
})
})

request
.on('error', (err) => {
callback(err, null, null)
})
.on('error', (err) => {
callback(err, null, null)
})

if (requestArgs.action == 'upload') {
try {
request.write(fs.readFileSync(requestArgs.source))
} catch (err) {
callback(err, null, null)
}
}
// Handle file stream errors
fileStream.on('error', (error) => {
console.error('File stream error:', error.message);
req.destroy();
reject(error);
});

request.end()
// Pipe the file to the request
fileStream.pipe(request);
} else {
request.end();
}
}

validatePath(path) {
Expand Down
9 changes: 2 additions & 7 deletions lib/netstorage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,6 @@ class Netstorage {
if (!(this.netstorageOpts.hostname && this.netstorageOpts.keyName && this.netstorageOpts.key)) {
throw new Error('[Netstorage Error] You should input netstorage hostname, keyname and key all')
}
if (this.netstorageOpts.ssl === undefined) {
this.netstorageOpts.ssl = false
} else if (typeof(this.netstorageOpts.ssl) !== 'boolean') {
throw new TypeError('[Netstorage Error] "ssl" argument should be boolean type')
}

this.requestor = new APIRequest(this.netstorageOpts)

Expand All @@ -59,7 +54,7 @@ class Netstorage {
if (ns_path.endsWith('/')) {
return callback(new Error('[Netstorage Error] cannot download a directory'), null, null)
}
if (typeof(local_destination) === 'function' && callback === undefined) {
if (typeof (local_destination) === 'function' && callback === undefined) {
callback = local_destination
local_destination = ''
}
Expand Down Expand Up @@ -188,7 +183,7 @@ class Netstorage {
if (opts.path) {
if (opts.actions instanceof Object && Object.keys(opts.actions).length > 0) {
return {
action: baseActions+this.buildRequestActions(opts.actions),
action: baseActions + this.buildRequestActions(opts.actions),
method: 'GET',
path: opts.path
}
Expand Down
13 changes: 6 additions & 7 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,13 @@
"author": "Astin Choi <asciineo@gmail.com>",
"license": "Apache-2.0",
"devDependencies": {
"chai": "^3.5.0",
"mocha": "^3.2.0",
"chai": "^6.0.1",
"mocha": "^11.7.1",
"pre-commit": "^1.2.2"
},
"dependencies": {
"https-proxy-agent": "^2.1.1",
"lodash": "^4.17.4",
"xml2js": "^0.4.17",
"http-proxy-agent": "^2.1.0"
"https-proxy-agent": "^7.0.6",
"lodash": "^4.17.21",
"xml2js": "^0.6.2"
}
}
}