From 713db46b95fd8055bd9263c3b0c96320eb3b75e6 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Wed, 3 Apr 2019 21:25:17 -0700 Subject: [PATCH 1/3] add a Docker image with benchmarking utilities --- docker/tidb-bench/Dockerfile | 41 +++++++++++++++++++++++++++ docker/tidb-bench/README.md | 54 ++++++++++++++++++++++++++++++++++++ docker/tidb-bench/build.sh | 1 + 3 files changed, 96 insertions(+) create mode 100644 docker/tidb-bench/Dockerfile create mode 100644 docker/tidb-bench/README.md create mode 100755 docker/tidb-bench/build.sh diff --git a/docker/tidb-bench/Dockerfile b/docker/tidb-bench/Dockerfile new file mode 100644 index 0000000..66a2dde --- /dev/null +++ b/docker/tidb-bench/Dockerfile @@ -0,0 +1,41 @@ +FROM golang:1.12.1-alpine3.9 AS goycsb + +RUN apk add git build-base \ + && git clone https://github.com/pingcap/go-ycsb $GOPATH/src/github.com/pingcap/go-ycsb \ + && cd $GOPATH/src/github.com/pingcap/go-ycsb \ + && make + +FROM alpine:3.9 +MAINTAINER PingCAP + +# Install sysbench +# gcc and mariadb-dev install shared libraries +RUN apk add gcc mariadb-dev \ + && apk add --virtual .build-deps git build-base automake autoconf libtool --update \ + && git clone https://github.com/akopytov/sysbench.git \ + && cd sysbench \ + && ./autogen.sh \ + && ./configure --disable-shared \ + && make \ + && make install \ + && cd .. \ + && rm -r sysbench \ + && apk del .build-deps + +# TiDB patch for sysbench data loading (prepare) +RUN apk add --virtual .build-deps wget \ + && wget https://raw.githubusercontent.com/pingcap/tidb-bench/master/sysbench-patch/oltp_common.lua \ + && apk del .build-deps \ + && mv oltp_common.lua /usr/local/share/sysbench/oltp_common.lua + +RUN apk add mysql-client +RUN apk add bash +COPY --from=dmonakhov/alpine-fio /usr/local/bin/fio /usr/local/bin/fio + +COPY --from=goycsb /go/src/github.com/pingcap/go-ycsb/bin/go-ycsb /go-ycsb/bin/go-ycsb +COPY --from=goycsb /go/src/github.com/pingcap/go-ycsb/workloads /go-ycsb/workloads +COPY --from=goycsb /go/src/github.com/pingcap/go-ycsb/tool /go-ycsb/tool + +ADD README.md /README.md + +CMD ["bash"] diff --git a/docker/tidb-bench/README.md b/docker/tidb-bench/README.md new file mode 100644 index 0000000..6cba0e9 --- /dev/null +++ b/docker/tidb-bench/README.md @@ -0,0 +1,54 @@ +This image includes various utilities used to benchmark TiDB. + +# Performance tests + +* sysbench +* go-ycsb + + +# Disk tests + +*Note:* All the commands need to be run under the disk mount point. And the test.data file need to be deleted before test. + +## fio +* Read +``` +fio -ioengine=libaio -bs=32k -direct=1 -thread -rw=read -size=10G -filename=test.data -name="PingCAP max throughput" -iodepth=4 -runtime=60 +``` + +* Write +``` +fio -ioengine=libaio -bs=32k -direct=1 -thread -rw=write -size=10G -filename=test.data -name="PingCAP max throughput" -iodepth=4 -runtime=60 +``` + +* Random read +``` +fio -ioengine=libaio -bs=32k -direct=1 -thread -rw=randread -size=10G -filename=test.data -name="PingCAP max throughput" -iodepth=4 -runtime=60 +``` + +* Random write +``` +fio -ioengine=libaio -bs=32k -direct=1 -thread -rw=randwrite -size=10G -filename=test.data -name="PingCAP max throughput" -iodepth=4 -runtime=60 +``` + +* Random read & write +``` +fio -ioengine=libaio -bs=32k -direct=1 -thread -rw=randrw -percentage_random=100,0 -size=10G -filename=test.data -name="PingCAP max throughput" -iodepth=4 -runtime=60 +``` + +## dd + +* Cache write +``` +dd bs=4k count=400000 if=/dev/zero of=test.data +``` + +* Direct write +``` +dd bs=4k count=400000 if=/dev/zero of=test.data oflag=direct +``` + +* Direct read +``` +dd bs=4k count=400000 if=test.data of=/dev/null& oflag=direct +``` diff --git a/docker/tidb-bench/build.sh b/docker/tidb-bench/build.sh new file mode 100755 index 0000000..3780abc --- /dev/null +++ b/docker/tidb-bench/build.sh @@ -0,0 +1 @@ +docker build -t pingcap/tidb-bench . From 6e7364f0ea96911718991b9f0ccce0d26761f638 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Thu, 4 Apr 2019 08:40:45 -0700 Subject: [PATCH 2/3] separate out a tidb-sysbench image --- docker/tidb-bench/Dockerfile | 23 ++--------------------- docker/tidb-bench/build.sh | 5 +++++ docker/tidb-sysbench/Dockerfile | 28 ++++++++++++++++++++++++++++ docker/tidb-sysbench/build.sh | 3 +++ 4 files changed, 38 insertions(+), 21 deletions(-) create mode 100644 docker/tidb-sysbench/Dockerfile create mode 100755 docker/tidb-sysbench/build.sh diff --git a/docker/tidb-bench/Dockerfile b/docker/tidb-bench/Dockerfile index 66a2dde..7832a57 100644 --- a/docker/tidb-bench/Dockerfile +++ b/docker/tidb-bench/Dockerfile @@ -8,28 +8,9 @@ RUN apk add git build-base \ FROM alpine:3.9 MAINTAINER PingCAP -# Install sysbench -# gcc and mariadb-dev install shared libraries -RUN apk add gcc mariadb-dev \ - && apk add --virtual .build-deps git build-base automake autoconf libtool --update \ - && git clone https://github.com/akopytov/sysbench.git \ - && cd sysbench \ - && ./autogen.sh \ - && ./configure --disable-shared \ - && make \ - && make install \ - && cd .. \ - && rm -r sysbench \ - && apk del .build-deps +COPY --from=pingcap/tidb-sysbench /usr/local/bin/sysbench /usr/local/bin/sysbench +COPY --from=pingcap/tidb-sysbench /usr/local/share/sysbench /usr/local/share/sysbench -# TiDB patch for sysbench data loading (prepare) -RUN apk add --virtual .build-deps wget \ - && wget https://raw.githubusercontent.com/pingcap/tidb-bench/master/sysbench-patch/oltp_common.lua \ - && apk del .build-deps \ - && mv oltp_common.lua /usr/local/share/sysbench/oltp_common.lua - -RUN apk add mysql-client -RUN apk add bash COPY --from=dmonakhov/alpine-fio /usr/local/bin/fio /usr/local/bin/fio COPY --from=goycsb /go/src/github.com/pingcap/go-ycsb/bin/go-ycsb /go-ycsb/bin/go-ycsb diff --git a/docker/tidb-bench/build.sh b/docker/tidb-bench/build.sh index 3780abc..90a404f 100755 --- a/docker/tidb-bench/build.sh +++ b/docker/tidb-bench/build.sh @@ -1 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +pushd ../tidb-sysbench +./build.sh +popd docker build -t pingcap/tidb-bench . diff --git a/docker/tidb-sysbench/Dockerfile b/docker/tidb-sysbench/Dockerfile new file mode 100644 index 0000000..3ba2344 --- /dev/null +++ b/docker/tidb-sysbench/Dockerfile @@ -0,0 +1,28 @@ +FROM alpine:3.9 +MAINTAINER PingCAP + +# Install sysbench +# gcc and mariadb-dev install shared libraries +RUN apk add gcc mariadb-dev \ + && apk add --virtual .build-deps git build-base automake autoconf libtool --update \ + && git clone https://github.com/akopytov/sysbench.git \ + && cd sysbench \ + && ./autogen.sh \ + && ./configure --disable-shared \ + && make \ + && make install \ + && cd .. \ + && rm -r sysbench \ + && apk del .build-deps + +# TiDB patch for sysbench data loading (prepare) +RUN apk add --virtual .build-deps wget \ + && wget https://raw.githubusercontent.com/pingcap/tidb-bench/master/sysbench-patch/oltp_common.lua \ + && apk del .build-deps \ + && mv oltp_common.lua /usr/local/share/sysbench/oltp_common.lua + +RUN apk add mysql-client +RUN apk add bash + +WORKDIR "/usr/local/share/sysbench" +CMD ["sysbench"] diff --git a/docker/tidb-sysbench/build.sh b/docker/tidb-sysbench/build.sh new file mode 100755 index 0000000..2126316 --- /dev/null +++ b/docker/tidb-sysbench/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +docker build -t pingcap/tidb-sysbench . From 77da425a1fd18ea8712f93c3a04d9f989e82efd6 Mon Sep 17 00:00:00 2001 From: Greg Weber Date: Thu, 4 Apr 2019 12:09:23 -0700 Subject: [PATCH 3/3] use sysbench version 1.0 --- docker/tidb-sysbench/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/tidb-sysbench/Dockerfile b/docker/tidb-sysbench/Dockerfile index 3ba2344..4a330c7 100644 --- a/docker/tidb-sysbench/Dockerfile +++ b/docker/tidb-sysbench/Dockerfile @@ -2,11 +2,13 @@ FROM alpine:3.9 MAINTAINER PingCAP # Install sysbench -# gcc and mariadb-dev install shared libraries -RUN apk add gcc mariadb-dev \ - && apk add --virtual .build-deps git build-base automake autoconf libtool --update \ +# gcc and mariadb-dev install shared libraries used at runtime +RUN apk add gcc mariadb-dev + +RUN apk add --virtual .build-deps git build-base automake autoconf libtool --update \ && git clone https://github.com/akopytov/sysbench.git \ && cd sysbench \ + && git checkout 1.0.17 \ && ./autogen.sh \ && ./configure --disable-shared \ && make \ @@ -19,7 +21,8 @@ RUN apk add gcc mariadb-dev \ RUN apk add --virtual .build-deps wget \ && wget https://raw.githubusercontent.com/pingcap/tidb-bench/master/sysbench-patch/oltp_common.lua \ && apk del .build-deps \ - && mv oltp_common.lua /usr/local/share/sysbench/oltp_common.lua + && mv oltp_common.lua /usr/local/share/sysbench/oltp_common.lua \ + && chmod +x /usr/local/share/sysbench/oltp_common.lua RUN apk add mysql-client RUN apk add bash