setup travis-ci with test-twemproxy

.gitignore

@@ -1,3 +1,6 @@
+# pyc
+*.pyc
+
 # Compiled Object files
 *.lo
 *.o

.travis.yml

@@ -1,3 +1,3 @@
 language: c
-script: CFLAGS="-ggdb3 -O0" autoreconf -fvi && ./configure --enable-debug=log && make && sudo make install
+script: bash ./travis.sh
 

notes/redis.md

@@ -79,9 +79,7 @@
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |      MGET         |    Yes     | MGET key [key ...]                                                                                                  |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      MSET         |    No      | MSET key value [key value ...]                                                                                      |
-    +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      MSETNX       |    No      | MSETNX key value [key value ...]                                                                                    |
+    |      MSET         |    Yes*    | MSET key value [key value ...]                                                                                      | 
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |      PSETEX       |    Yes     | PSETEX key milliseconds value                                                                                       |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
@@ -98,6 +96,8 @@
     |      STRLEN       |    Yes     | STRLEN key                                                                                                          |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
 
+* MSET support is not Atomic 
+
 ### Hashes
 
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+

scripts/benchmark-mget.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+#coding: utf-8
+#file   : test_mget.py
+#author : ning
+#date   : 2014-04-01 13:15:48
+
+import os
+import re
+import commands
+
+ports = [
+    4001,  # before improve
+    4000,  # after improve
+    2000   # redis
+]
+
+def system(cmd):
+    return commands.getoutput(cmd)
+
+def extra(regex, text):
+    match = re.search(regex, text, re.DOTALL)
+    if match:
+        return match.group(1)
+
+def testit():
+    for mget_size in [10, 100, 1000, 10000]:
+        for port in ports:
+            cnt = 100*1000 / mget_size
+            clients = 50
+            if mget_size == 10000:
+                clients = 2
+            cmd = 'cd /home/ning/xredis/deploy-srcs/redis-2.8.3/src && ./redis-benchmark.%d -n %d -p %d -t mget  -r 1000000000 -c %d' % (mget_size, cnt, port, clients)
+            #print cmd
+            rst = system(cmd)
+
+            #100.00% <= 2 milliseconds
+            #28089.89 requests per second
+            rtime = extra('100.00% <= (\d+) milliseconds', rst)
+            qps = extra('([\.\d]+) requests per second', rst)
+
+            print 'mget_size=%d on %d: pqs: %s, rtime: %s' % (mget_size, port, qps, rtime)
+
+testit()

scripts/test_mget_mset.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+#coding: utf-8
+#file   : test_mget.py
+#author : ning
+#date   : 2014-04-01 13:15:48
+
+'''
+usage:
+
+export REDIS_HOST=127.0.0.1
+export REDIS_PORT=4000
+nosetests
+
+'''
+
+import os
+import redis
+
+host = os.environ['REDIS_HOST']
+port = int(os.environ['REDIS_PORT'])
+
+
+def test_mget(cnt=10):
+    print 'test_many', cnt
+    r = redis.StrictRedis(host, port)
+
+    def insert_by_pipeline():
+        pipe = r.pipeline(transaction=False)
+        for i in range(cnt):
+            pipe.set('kkk-%s'%i, 'vvv-%s'%i)
+        pipe.execute()
+
+    def insert_by_mset():
+        kv = {'kkk-%s' % i :'vvv-%s' % i for i in range(cnt)}
+        ret = r.mset(**kv)
+
+    insert_by_mset()
+    keys = ['kkk-%s' % i for i in range(cnt)]
+
+    #mget to check
+    vals = r.mget(keys)
+    #print 'vals', vals
+    for i in range(cnt):
+        assert('vvv-%s'%i == vals[i])
+
+    #del
+    assert (cnt == r.delete(*keys) )
+
+    #mget again
+    vals = r.mget(keys)
+    for i in range(cnt):
+        assert(None == vals[i])
+
+def test_many_mget():
+    for i in range(1, 10000, 17):
+        test_mget(i)
+    pass
+
+def test_large_mget(cnt=5):
+    r = redis.StrictRedis(host, port)
+
+    kv = {}
+    for i in range(cnt):
+        kv['kkx-%s' % i] = os.urandom(1024*1024*8)
+
+    #insert
+    for i in range(cnt):
+        key = 'kkx-%s' % i
+        r.set(key, kv[key])
+
+    keys = ['kkx-%s' % i for i in range(cnt)]
+
+    #mget to check
+    vals = r.mget(keys)
+    for i in range(cnt):
+        key = 'kkx-%s' % i
+        assert(kv[key] == vals[i])
+
+

src/nc_core.c

@@ -172,9 +172,9 @@ core_send(struct context *ctx, struct conn *conn)
 
     status = conn->send(ctx, conn);
     if (status != NC_OK) {
-        log_debug(LOG_INFO, "send on %c %d failed: %s",
+        log_debug(LOG_INFO, "send on %c %d failed: status:%d errno: %d %s",
                   conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd,
-                  strerror(errno));
+                  status, errno, strerror(errno));
     }
 
     return status;

src/nc_log.c

@@ -250,5 +250,12 @@ _log_hexdump(const char *file, int line, char *data, int datalen,
         l->nerror++;
     }
 
+    if (len >= size - 1){
+        n = nc_write(l->fd, "\n", 1);
+        if (n < 0) {
+            l->nerror++;
+        }
+    }
+
     errno = errno_save;
 }

src/nc_log.h

@@ -38,7 +38,7 @@ struct logger {
 #define LOG_VVVERB  10  /* verbose messages on ganga */
 #define LOG_PVERB   11  /* periodic verbose messages on crack */
 
-#define LOG_MAX_LEN 256 /* max length of log message */
+#define LOG_MAX_LEN 512 /* max length of log message */
 
 /*
  * log_stderr   - log to stderr