From dacd6136e38ac155686a0825028bd0313c356046 Mon Sep 17 00:00:00 2001 From: FisehaTessema4 <60725293+FisehaTessema4@users.noreply.github.com> Date: Tue, 8 Mar 2022 23:30:22 -0800 Subject: [PATCH 1/5] Add files via upload --- docker-compose.yml | 48 +++++++++++++++++++++++++++++++ example.cnf | 72 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 docker-compose.yml create mode 100644 example.cnf diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5f08adb6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,48 @@ +version: '2' +services: + primary: + image: mariadb:latest + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: 'Y' + volumes: + - ./sql/primary:/docker-entrypoint-initdb.d + command: mysqld --log-bin=mariadb-bin --binlog-format=ROW --server-id=3000 + ports: + - "4001:3306" + + replica1: + image: mariadb:latest + depends_on: + - primary + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: 'Y' + volumes: + - ./sql/replica:/docker-entrypoint-initdb.d + command: mysqld --log-bin=mariadb-bin --binlog-format=ROW --server-id=3001 --log-slave-updates + ports: + - "4002:3306" + + replica2: + image: mariadb:latest + depends_on: + - primary + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: 'Y' + volumes: + - ./sql/replica:/docker-entrypoint-initdb.d + command: mysqld --log-bin=mariadb-bin --binlog-format=ROW --server-id=3002 --log-slave-updates + ports: + - "4003:3306" + + maxscale: + image: mariadb/maxscale:latest + depends_on: + - primary + - replica1 + - replica2 + volumes: + - ./maxscale.cnf.d:/etc/maxscale.cnf.d + ports: + - "4006:4006" # readwrite port + - "4008:4008" # readonly port + - "8989:8989" # REST API port diff --git a/example.cnf b/example.cnf new file mode 100644 index 00000000..9e072073 --- /dev/null +++ b/example.cnf @@ -0,0 +1,72 @@ +[server1] +type=server +address=primary +port=3306 +protocol=MariaDBBackend + +[server2] +type=server +address=replica1 +port=3306 +protocol=MariaDBBackend + +[server3] +type=server +address=replica2 +port=3306 +protocol=MariaDBBackend + +# Monitor for the servers +# This will keep MaxScale aware of the state of the servers. +# MySQL Monitor documentation: +# https://github.com/mariadb-corporation/MaxScale/blob/2.3/Documentation/Monitors/MariaDB-Monitor.md + +[MariaDB-Monitor] +type=monitor +module=mariadbmon +servers=server1,server2,server3 +user=maxuser +password=maxpwd +auto_failover=true +auto_rejoin=true +enforce_read_only_slaves=1 + +# Service definitions +# Service Definition for a read-only service and a read/write splitting service. + +# ReadConnRoute documentation: +# https://github.com/mariadb-corporation/MaxScale/blob/2.3/Documentation/Routers/ReadConnRoute.md + +[Read-Only-Service] +type=service +router=readconnroute +servers=server1,server2,server3 +user=maxuser +password=maxpwd +router_options=slave + +# ReadWriteSplit documentation: +# https://github.com/mariadb-corporation/MaxScale/blob/2.3/Documentation/Routers/ReadWriteSplit.md + +[Read-Write-Service] +type=service +router=readwritesplit +servers=server1,server2,server3 +user=maxuser +password=maxpwd +master_failure_mode=fail_on_write + +# Listener definitions for the services +# Listeners represent the ports the services will listen on. + +[Read-Only-Listener] +type=listener +service=Read-Only-Service +protocol=MySQLClient +port=4008 + +[Read-Write-Listener] +type=listener +service=Read-Write-Service +protocol=MySQLClient +port=4006 From 7913a62b1a039e148f00d40ee03f64d7d352de4b Mon Sep 17 00:00:00 2001 From: FisehaTessema4 <60725293+FisehaTessema4@users.noreply.github.com> Date: Thu, 10 Mar 2022 20:33:45 -0800 Subject: [PATCH 2/5] Add files via upload Added a python file ,Write a python script that will talk to the Maxscale instance, perform the following queries, and print the output to the console: The last 10 rows of zipcodes_one The first 10 rows of zipcodes_two The largest zipcode in zipcodes_one The smallest zipcode in zipcodes_two --- python.txt | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 python.txt diff --git a/python.txt b/python.txt new file mode 100644 index 00000000..d710ef9d --- /dev/null +++ b/python.txt @@ -0,0 +1,33 @@ +import pymysql + +db=pymysql connector(host=“172.28.0.4”,port=4000, user=“Maxuser”,passwd=“mapped”) +Cursor = db.cursor() + +print(“The last 10 rows of zipcodes_one are:”) +Cursor.execute(“SELECT*FROM zipcodes_one zipcodes_one LIMIT 9990,10.”) +Results = cursor.fetchall() +For result in results +print(result) + + + +print(The last 10 rows of zipcodes_two are:) +Cursor.execute(“SELECT*FROM zipcodes_twozipcodes_two LIMIT 10”) +Results = cursor.fetchall() +For result in results +print(result) + + +print(“The largest zipcode zipcode number in zipcodes_one is :“) +Cursor = db.cursor() +Cursor.execute(“SELECT zipcode *FROM zipcodes_onezipcodes_one ORDER BY zip code DESC LIMIT1”) +Results = cursor.fetchall() +For result in results +print(result) + +print(The Smallest zipcode zipcode number in zipcodes_two is :“) +Cursor = db.cursor() +Cursor.execute(“SELECT zipcode FROM zipcodes_one zipcodes_one ORDER BY zip code ASC LIMIT1”) +Results = cursor.fetchall() +For result in results +print(result) From 7c3dad140efdfe701afd57fc74fcf34292c9ddc5 Mon Sep 17 00:00:00 2001 From: FisehaTessema4 <60725293+FisehaTessema4@users.noreply.github.com> Date: Mon, 14 Mar 2022 13:57:00 -0700 Subject: [PATCH 3/5] Update python.txt --- python.txt | 59 ++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/python.txt b/python.txt index d710ef9d..2ee3fd87 100644 --- a/python.txt +++ b/python.txt @@ -1,33 +1,30 @@ import pymysql -db=pymysql connector(host=“172.28.0.4”,port=4000, user=“Maxuser”,passwd=“mapped”) -Cursor = db.cursor() - -print(“The last 10 rows of zipcodes_one are:”) -Cursor.execute(“SELECT*FROM zipcodes_one zipcodes_one LIMIT 9990,10.”) -Results = cursor.fetchall() -For result in results -print(result) - - - -print(The last 10 rows of zipcodes_two are:) -Cursor.execute(“SELECT*FROM zipcodes_twozipcodes_two LIMIT 10”) -Results = cursor.fetchall() -For result in results -print(result) - - -print(“The largest zipcode zipcode number in zipcodes_one is :“) -Cursor = db.cursor() -Cursor.execute(“SELECT zipcode *FROM zipcodes_onezipcodes_one ORDER BY zip code DESC LIMIT1”) -Results = cursor.fetchall() -For result in results -print(result) - -print(The Smallest zipcode zipcode number in zipcodes_two is :“) -Cursor = db.cursor() -Cursor.execute(“SELECT zipcode FROM zipcodes_one zipcodes_one ORDER BY zip code ASC LIMIT1”) -Results = cursor.fetchall() -For result in results -print(result) +db = pymysql.connect(host="172.20.0.4", port=4000, user="maxuser", passwd="maxpwd") +cursor = db.cursor() + +print('The last 10 rows of zipcodes_one are:') +cursor.execute("SELECT * FROM zipcodes_one.zipcodes_one LIMIT 9990,10;") +results = cursor.fetchall() +for result in results: + print(result) + +print('The first 10 rows of zipcodes_two are:') +cursor.execute("SELECT * FROM zipcodes_two.zipcodes_two LIMIT 10") +results = cursor.fetchall() +for result in results: + print(result) + +print('The largest zipcode number in zipcodes_one is:') +cursor = db.cursor() +cursor.execute("SELECT Zipcode FROM zipcodes_one.zipcodes_one ORDER BY Zipcode DESC LIMIT 1;") +results = cursor.fetchall() +for result in results: + print(result) + +print('The smallest zipcode number in zipcodes_two is:') +cursor = db.cursor() +cursor.execute("SELECT Zipcode FROM zipcodes_two.zipcodes_two ORDER BY Zipcode ASC LIMIT 1;") +results = cursor.fetchall() +for result in results: + print(result) From 38e595cd6fd56a8b9b7c9cfea3213291fe353b2b Mon Sep 17 00:00:00 2001 From: FisehaTessema4 <60725293+FisehaTessema4@users.noreply.github.com> Date: Mon, 14 Mar 2022 14:03:54 -0700 Subject: [PATCH 4/5] Update README.md --- README.md | 186 ++++++++++++++++++++++++++---------------------------- 1 file changed, 90 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 6fc22a1d..bc3d03db 100644 --- a/README.md +++ b/README.md @@ -1,131 +1,125 @@ -# MariaDB MaxScale Docker image +# Real World Project CNE 370 -This Docker image runs the latest 2.4 version of MariaDB MaxScale. -- [Travis CI: - ![build status badge](https://img.shields.io/travis/mariadb-corporation/maxscale-docker/master.svg)](https://travis-ci.org/mariadb-corporation/maxscale-docker/branches) +## Overview -## Building +This project utilizes docker-compose containers running MaxScale to create a sharded SQL database containing zipcode information. A python script is then used to connect, query, and demonstrate the merged database. -Run the following command in this directory to build the image. +## Configuration + +### docker-compose.yml + +This file is used as a blueprint to build your docker containers. You can specify how many and what type of containers you want to build. + +### maxscale.cnf + +This file configures your MaxScale instance. Docker-compose.yml calls upon this file to build the MaxScale container. + +### Shard_query.py + +This is the Python script. You can configure it to retrieve specific data from the database. + +## Building Container and Running Code + +Go into the /maxscale directory and run the following to build your containers ``` -make build-image +sudo docker-compose up -d ``` -## Running -To pull the latest MaxScale image from docker hub: +Confirm that your containers have been successfully built. You should see the containers you've specified in docker-compose.yml. ``` -docker pull mariadb/maxscale:latest +sudo docker-compose ps ``` -To run the MaxScale container overriding the container instance name to 'mxs': +Install python3 and pymysql + ``` -docker run -d --name mxs mariadb/maxscale:latest +sudo apt-get install python3.6 +sudo apt-get install python3-pymysql ``` -Read on for details of how to configure the MaxScale container. - -## Configuration -The default configuration for the container is fairly minimalist and can be found in [this configuration file](./maxscale.cnf). At a high level the following is enabled: -- REST API with default user and password (admin / mariadb) listening to all hosts (0.0.0.0) +Go into the /Real_world_project_CNE370 directory and run the following to execute the python script -### Configure via REST API -The REST API by default listens on port 8989. To interact with this from the docker host, requires a port mapping to specified on container startup. The example below shows listing the current services via curl: ``` -docker run -d -p 8989:8989 --name mxs mariadb/maxscale:latest -curl -u admin:mariadb -H "Content-Type: application/json" http://localhost:8989/v1/services - +python3 zipcode.py ``` -### Configure via maxscale.cnf File -An alternative model is to provide an overlay maxscale.cnf file that provides additional configuration for the cluster to be managed. To do this, you must mount your configuration file into `/etc/maxscale.cnf.d/`. When running the container with docker directly pass this using the argument to the `-v` option: +Results ``` -docker run -d --name mxs -v $PWD/my-maxscale.cnf:/etc/maxscale.cnf.d/my-maxscale.cnf mariadb/maxscale:2.2 +The last 10 rows of zipcodes_one are: +(40843, 'STANDARD', 'HOLMES MILL', 'KY', 'PRIMARY', '36.86', '-83', 'NA-US-KY-HOLMES MILL', 'FALSE', '', '', '') +(41425, 'STANDARD', 'EZEL', 'KY', 'PRIMARY', '37.89', '-83.44', 'NA-US-KY-EZEL', 'FALSE', '390', '801', '10204009') +(40118, 'STANDARD', 'FAIRDALE', 'KY', 'PRIMARY', '38.11', '-85.75', 'NA-US-KY-FAIRDALE', 'FALSE', '4398', '7635', '122449930') +(40020, 'PO BOX', 'FAIRFIELD', 'KY', 'PRIMARY', '37.93', '-85.38', 'NA-US-KY-FAIRFIELD', 'FALSE', '', '', '') +(42221, 'PO BOX', 'FAIRVIEW', 'KY', 'PRIMARY', '36.84', '-87.31', 'NA-US-KY-FAIRVIEW', 'FALSE', '', '', '') +(41426, 'PO BOX', 'FALCON', 'KY', 'PRIMARY', '37.78', '-83', 'NA-US-KY-FALCON', 'FALSE', '', '', '') +(40932, 'PO BOX', 'FALL ROCK', 'KY', 'PRIMARY', '37.22', '-83.78', 'NA-US-KY-FALL ROCK', 'FALSE', '', '', '') +(40119, 'STANDARD', 'FALLS OF ROUGH', 'KY', 'PRIMARY', '37.6', '-86.55', 'NA-US-KY-FALLS OF ROUGH', 'FALSE', '760', '1468', '20771670') +(42039, 'STANDARD', 'FANCY FARM', 'KY', 'PRIMARY', '36.75', '-88.79', 'NA-US-KY-FANCY FARM', 'FALSE', '696', '1317', '20643485') +(40319, 'PO BOX', 'FARMERS', 'KY', 'PRIMARY', '38.14', '-83.54', 'NA-US-KY-FARMERS', 'FALSE', '', '', '') +The first 10 rows of zipcodes_two are: +(42040, 'STANDARD', 'FARMINGTON', 'KY', 'PRIMARY', '36.67', '-88.53', 'NA-US-KY-FARMINGTON', 'FALSE', '465', '896', '11562973') +(41524, 'STANDARD', 'FEDSCREEK', 'KY', 'PRIMARY', '37.4', '-82.24', 'NA-US-KY-FEDSCREEK', 'FALSE', '', '', '') +(42533, 'STANDARD', 'FERGUSON', 'KY', 'PRIMARY', '37.06', '-84.59', 'NA-US-KY-FERGUSON', 'FALSE', '429', '761', '9555412') +(40022, 'STANDARD', 'FINCHVILLE', 'KY', 'PRIMARY', '38.15', '-85.31', 'NA-US-KY-FINCHVILLE', 'FALSE', '437', '839', '19909942') +(40023, 'STANDARD', 'FISHERVILLE', 'KY', 'PRIMARY', '38.16', '-85.42', 'NA-US-KY-FISHERVILLE', 'FALSE', '1884', '3733', '113020684') +(41743, 'PO BOX', 'FISTY', 'KY', 'PRIMARY', '37.33', '-83.1', 'NA-US-KY-FISTY', 'FALSE', '', '', '') +(41219, 'STANDARD', 'FLATGAP', 'KY', 'PRIMARY', '37.93', '-82.88', 'NA-US-KY-FLATGAP', 'FALSE', '708', '1397', '20395667') +(40935, 'STANDARD', 'FLAT LICK', 'KY', 'PRIMARY', '36.82', '-83.76', 'NA-US-KY-FLAT LICK', 'FALSE', '752', '1477', '14267237') +(40997, 'STANDARD', 'WALKER', 'KY', 'PRIMARY', '36.88', '-83.71', 'NA-US-KY-WALKER', 'FALSE', '', '', '') +(41139, 'STANDARD', 'FLATWOODS', 'KY', 'PRIMARY', '38.51', '-82.72', 'NA-US-KY-FLATWOODS', 'FALSE', '3692', '6748', '121902277') +The largest zipcode number in zipcodes_one is: +(47750,) +The smallest zipcode number in zipcodes_two is: +(38257,) + ``` -## MaxScale docker-compose setup -[The MaxScale docker-compose setup](./docker-compose.yml) contains MaxScale -configured with a three node master-slave cluster. To start it, run the -following commands in this directory. -``` -docker-compose build -docker-compose up -d -``` -After MaxScale and the servers have started (takes a few minutes), you can find -the readwritesplit router on port 4006 and the readconnroute on port 4008. The -user `maxuser` with the password `maxpwd` can be used to test the cluster. -Assuming the mariadb client is installed on the host machine: -``` -$ mysql -umaxuser -pmaxpwd -h 127.0.0.1 -P 4006 test -Welcome to the MariaDB monitor. Commands end with ; or \g. -Your MySQL connection id is 5 -Server version: 10.2.12 2.2.9-maxscale mariadb.org binary distribution -Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. -Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. -MySQL [test]> -``` -You can edit the [`maxscale.cnf.d/example.cnf`](./maxscale.cnf.d/example.cnf) -file and recreate the MaxScale container to change the configuration. -To stop the containers, execute the following command. Optionally, use the -v -flag to also remove the volumes. -To run maxctrl in the container to see the status of the cluster: -``` -$ docker-compose exec maxscale maxctrl list servers -┌─────────┬─────────┬──────┬─────────────┬─────────────────┬──────────┐ -│ Server │ Address │ Port │ Connections │ State │ GTID │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼──────────┤ -│ server1 │ master │ 3306 │ 0 │ Master, Running │ 0-3000-5 │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼──────────┤ -│ server2 │ slave1 │ 3306 │ 0 │ Slave, Running │ 0-3000-5 │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼──────────┤ -│ server3 │ slave2 │ 3306 │ 0 │ Running │ 0-3000-5 │ -└─────────┴─────────┴──────┴─────────────┴─────────────────┴──────────┘ -``` -The cluster is configured to utilize automatic failover. To illustrate this you can stop the master -container and watch for maxscale to failover to one of the original slaves and then show it rejoining -after recovery: -``` -$ docker-compose stop master -Stopping maxscaledocker_master_1 ... done -$ docker-compose exec maxscale maxctrl list servers -┌─────────┬─────────┬──────┬─────────────┬─────────────────┬─────────────┐ -│ Server │ Address │ Port │ Connections │ State │ GTID │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼─────────────┤ -│ server1 │ master │ 3306 │ 0 │ Down │ 0-3000-5 │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼─────────────┤ -│ server2 │ slave1 │ 3306 │ 0 │ Master, Running │ 0-3001-7127 │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼─────────────┤ -│ server3 │ slave2 │ 3306 │ 0 │ Slave, Running │ 0-3001-7127 │ -└─────────┴─────────┴──────┴─────────────┴─────────────────┴─────────────┘ -$ docker-compose start master -Starting master ... done -$ docker-compose exec maxscale maxctrl list servers -┌─────────┬─────────┬──────┬─────────────┬─────────────────┬─────────────┐ -│ Server │ Address │ Port │ Connections │ State │ GTID │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼─────────────┤ -│ server1 │ master │ 3306 │ 0 │ Slave, Running │ 0-3001-7127 │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼─────────────┤ -│ server2 │ slave1 │ 3306 │ 0 │ Master, Running │ 0-3001-7127 │ -├─────────┼─────────┼──────┼─────────────┼─────────────────┼─────────────┤ -│ server3 │ slave2 │ 3306 │ 0 │ Slave, Running │ 0-3001-7127 │ -└─────────┴─────────┴──────┴─────────────┴─────────────────┴─────────────┘ -``` -Once complete, to remove the cluster and maxscale containers: -``` -docker-compose down -v -``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 38e995abb51a0c680a83ab77581430e4d715a896 Mon Sep 17 00:00:00 2001 From: FisehaTessema4 <60725293+FisehaTessema4@users.noreply.github.com> Date: Mon, 14 Mar 2022 14:04:51 -0700 Subject: [PATCH 5/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc3d03db..bdaa8f05 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This file is used as a blueprint to build your docker containers. You can specif This file configures your MaxScale instance. Docker-compose.yml calls upon this file to build the MaxScale container. -### Shard_query.py +### zipcode.py This is the Python script. You can configure it to retrieve specific data from the database.