Add sharding support

.gitignore

@@ -29,4 +29,4 @@ build/Release
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
 
-test/db
+test/db.*

README.md

@@ -26,6 +26,22 @@ Put tokens:
 limitdctl --bucket ip --key 127.0.0.1 --put 1
 ```
 
+Get status:
+
+```
+limitdctl --bucket ip --key 127.0.0.1 --status
+```
+
+### Sharding
+ In order to use sharding with the cli you must specify `--shard` option and then
+ either `--hosts <list>` or `--autodiscover <hostname>` (see shard options for [node-client](https://github.com/limitd/node-client#sharding))
+
+Examples:
+```
+limitdctl --shard --hosts 192.222.222.222,192.222.222.223,192.222.222.224 --bucket ip --key 127.0.0.1 --status
+limitdctl --shard --autodiscover autodiscover.int.mylimitd.com --bucket ip --key 127.0.0.1 --put 1
+```
+
 ## License
 
-MIT 2015 - Auth0 Inc.
+MIT 2015 - Auth0 Inc.

bin/limitdctl

@@ -1,13 +1,19 @@
 #!/usr/bin/env node
-
 var program = require('commander');
 var LimitdClient = require('limitd-client');
 var _ = require('lodash');
 var Table = require('cli-table');
 
+function list(val) {
+  return typeof(val) === 'string' && val.split(',');
+}
+
 program.version(require('../package').version)
-  .option('-p, --port [9231]', 'Port to bind [9231].', '9231')
-  .option('-h, --hostname [0.0.0.0]', 'Hostname to bind [0.0.0.0].', '0.0.0.0')
+  .option('-p, --port [9231]', 'Port to bind [9231].')
+  .option('-h --hostname [0.0.0.0]', 'Hostname/s to bind.')
+  .option('-hs, --hosts <hosts>', 'Comma separated list of hosts', list)
+  .option('-sh, --shard', 'Use a shard client. You must specify either one or more hostnames or the autodiscover option')
+  .option('-a, --autodiscover <hostname>', 'DNS-based shard members discovery.')
   .option('-b, --bucket <bucket>', 'Bucket type')
   .option('-k, --key <key>', 'Bucket key')
   .option('-t, --take <n>', 'Take or fail n tokens from the bucket.', parseInt)
@@ -29,21 +35,65 @@ if (!program.key) {
 
 var methods = _.intersection(['take', 'wait', 'put', 'status'], Object.keys(program));
 if (methods.length > 1 || methods.length === 0) {
-  console.error('one and only one method must be provided "--take", "--wait" or "--put"');
+  console.error('one and only one method must be provided "--take", "--wait", "--put" or "--status"');
+  return process.exit(1);
+}
+
+var hostOptions = _.intersection(['autodiscover', 'hosts', 'hostname'], Object.keys(program));
+if (hostOptions.length > 1 || hostOptions.length === 0) {
+  console.error('one and only one option must be provided "--autodiscover", "--hosts", "--hostname"');
   return process.exit(1);
 }
 
+if (program.shard && !program.hosts && !program.autodiscover) {
+  console.error('"--shard" option requires "--hosts" or "--autodiscover" options to be provided');
+  return process.exit(1);
+}
+
+function normalizeLimitdConfig(program) {
+  if (program.autodiscover) {
+    return {
+      shard: {
+        autodiscover: { address: program.autodiscover }
+      }
+    };
+  }
+
+  if (Array.isArray(program.hosts)) {
+    if (program.shard) {
+      return {
+        shard: {
+          hosts: program.hosts
+        }
+      };
+    } else {
+      return { hosts: program.hosts };
+    }
+  }
+
+  return {
+    hosts: [
+      {
+        hostname: program.hostname,
+        port: program.port
+      }
+    ]
+  };
+}
+
 var method = methods[0];
-var config = _.pick(program, ['port', 'hostname']);
+var config = normalizeLimitdConfig(program);
+console.log(config);
 var client = new LimitdClient(config);
-
 client.once('ready', function () {
   if ( method === 'status' ) {
     client.status(program.bucket, program.key, function (err, result) {
       if (err) {
         console.error(err.message);
         return process.exit(1);
       }
+
+      console.log('aaaa!!')
       if (program.json) {
         console.log(JSON.stringify(result, null, 2));
       } else {

package.json

@@ -12,13 +12,13 @@
   },
   "dependencies": {
     "cli-table": "~0.3.1",
-    "commander": "~2.8.1",
-    "limitd-client": "~1.7.1",
+    "commander": "^2.11.0",
+    "limitd-client": "^2.12.0",
     "lodash": "~3.9.3"
   },
   "devDependencies": {
     "chai": "~3.0.0",
-    "limitd": "~4.2.0",
+    "limitd": "^5.3.1",
     "mocha": "~2.2.5",
     "rimraf": "~2.4.0"
   }

test/fixture.js

@@ -3,22 +3,18 @@ var rimraf = require('rimraf');
 var db = __dirname + '/db';
 
 module.exports = {
-  start: function (callback) {
+  start: function (port, db, callback) {
     rimraf.sync(db);
-    this.server = new LimitdServer({
+    const server = new LimitdServer({
       db: db,
+      port: port,
       buckets: {
         ip: {
           per_minute: 10
         }
       }
     });
-    this.server.start(callback);
-  },
-  stop: function () {
-    if (this.server) {
-      this.server.stop();
-      delete this.server;
-    }
+
+    server.start((err) => callback(err, server));
   }
 };

test/global.tests.js

@@ -1,3 +1,13 @@
 var fixture = require('./fixture');
-before(fixture.start);
-after(fixture.stop);
+var server;
+
+before((done) => {
+  fixture.start(9231, 'test/db.0', (err, iserver) => {
+    if (err) { return done(err); }
+
+    server = iserver;
+    done();
+  });
+});
+
+after(() => server && server.stop());

test/put.tests.js

@@ -12,5 +12,4 @@ describe('put', function () {
       });
     });
   });
-
-});
+});

test/shard.tests.js

@@ -0,0 +1,82 @@
+var exec = require('child_process').exec;
+var limitdctl = __dirname + '/../bin/limitdctl';
+var assert = require('chai').assert;
+var fixture = require('./fixture');
+
+describe('shard support', function() {
+  var server1;
+  var server2;
+
+  describe('using host options', () => {
+    before((done) => {
+      fixture.start(9232, 'test/db.1', (err, iserver1) => {
+        if (err) { return done(err); }
+
+        server1 = iserver1;
+
+        fixture.start(9233, 'test/db.2', (err, iserver2) => {
+          if (err) { return done(err); }
+
+          server2 = iserver2;
+
+          done();
+        });
+      });
+    });
+
+    after(() => {
+      server1.stop();
+      server2.stop();
+    });
+
+    const SHARD_PARAMS = '--shard --hosts limitd://127.0.0.1:9233,limitd://127.0.0.1:9232';
+
+    it('can put tokens to the shard', function (done) {
+      exec(limitdctl + ` ${SHARD_PARAMS} --bucket ip --key k1 --take 5`, function (err, stdout, stderr) {
+        if (err) { return done(err); }
+
+        exec(limitdctl + ` ${SHARD_PARAMS} --bucket ip --key k1 --put 2`, function (err, stdout, stderr) {
+          assert.equal(stdout, 'OK - Current: 7\n');
+          done(err);
+        });
+      });
+    });
+
+    it('can take tokens from the shard', function (done) {
+      exec(limitdctl + ` ${SHARD_PARAMS} --bucket ip --key k2 --take 1`, function (err, stdout, stderr) {
+        assert.equal(stdout, 'Conformant - Remaining: 9\n');
+        done(err);
+      });
+    });
+
+    it('can get status from the shard', function (done) {
+      this.timeout(4000)
+      exec(limitdctl + ` ${SHARD_PARAMS} --bucket ip --key k3-1 --take 1`, function (err, stdout, stderr) {
+        if (err) { return done(err); }
+
+        exec(limitdctl + ` ${SHARD_PARAMS} --bucket ip --key k3-2 --take 1`, function (err, stdout, stderr) {
+          if (err) { return done(err); }
+
+          exec(limitdctl + ` ${SHARD_PARAMS} --bucket ip --key k3- --status --json`, function (err, stdout, stderr) {
+            if (err) { return done(err); }
+
+            const parsed = JSON.parse(stdout).items;
+
+            assert.equal(parsed[0].instance, 'k3-1');
+            assert.equal(parsed[0].remaining, '9');
+            assert.equal(parsed[0].limit, '10');
+
+            assert.equal(parsed[1].instance, 'k3-2');
+            assert.equal(parsed[1].remaining, '9');
+            assert.equal(parsed[1].limit, '10');
+
+            done(err);
+          });
+        });
+      });
+    });
+  });
+
+  // It would be great to have integration tests for autodiscover but that
+  // would require a dns record
+});

test/take.tests.js

@@ -20,7 +20,7 @@ describe('take', function () {
 
   it('should fail if method is missing', function (done) {
     exec(limitdctl + ' --bucket ip --key 127.0.0.1', function (err, stdout, stderr) {
-      assert.include(stderr, 'one and only one method must be provided "--take", "--wait" or "--put"');
+      assert.include(stderr, 'one and only one method must be provided "--take", "--wait", "--put" or "--status"');
       done();
     });
   });
@@ -48,4 +48,4 @@ describe('take', function () {
     });
   });
 
-});
+});