From 1a725678b7a6f9d311536c46d1f8a7e5bebf6eab Mon Sep 17 00:00:00 2001 From: Everton Zanella Alvarenga Date: Mon, 25 Oct 2021 14:53:43 +0200 Subject: [PATCH 1/5] Dumping and restoring mysql database - using docker --- derex/runner/cli/mysql.py | 25 ++++++++++++++++++++++++- derex/runner/mysql.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/derex/runner/cli/mysql.py b/derex/runner/cli/mysql.py index 4462d326..1168ad9a 100644 --- a/derex/runner/cli/mysql.py +++ b/derex/runner/cli/mysql.py @@ -62,7 +62,6 @@ def drop(context: click.core.Context): def show(context: click.core.Context): """MySQL SHOW predicate""" - @create.command(name="database") @click.pass_obj @click.argument("db_name", type=str, required=False) @@ -243,3 +242,27 @@ def reset_mysql_password_cmd(current_password: str, force: bool): reset_mysql_password(current_password) return 0 + + +@mysql.command(name="dump") +@click.pass_obj +@click.argument("db_name", type=str) +def dump_database_cmd(project: Optional[Project], db_name: str): + """Dump a mysql database""" + + from derex.runner.mysql import dump_database + + dump_database(db_name) + return 0 + + +@mysql.command(name="restore") +@click.argument("db_name", type=str, nargs=1) +@click.argument("dump_file", type=click.Path(exists=True), nargs=1) +def restore_database_cmd(db_name: str, dump_file: str): + """Restore a mysql database from a file""" + + from derex.runner.mysql import restore_database + + restore_database(db_name, dump_file) + return 0 diff --git a/derex/runner/mysql.py b/derex/runner/mysql.py index 321aba20..331b8747 100644 --- a/derex/runner/mysql.py +++ b/derex/runner/mysql.py @@ -15,6 +15,7 @@ import logging import pymysql +import os logger = logging.getLogger(__name__) @@ -187,6 +188,34 @@ def copy_database(source_db_name: str, destination_db_name: str): ) +def dump_database(database_name: str): + """"Export the database""" + logger.info(f'Dumping the database "{database_name}"...') + # run_ddc_services( + # [ + # "exec", + # "-T", + # "mysql", + # f"mysqldump -u root -p{MYSQL_ROOT_PASSWORD} --databases {database_name} > dump.sql" + # ] + # ) + os.system( + f'docker exec mysql mysqldump -u {MYSQL_ROOT_USER} -p{MYSQL_ROOT_PASSWORD} {database_name} > {database_name}.sql') + logger.info( + f"The database {database_name} was successfully dumped" + ) + + +def restore_database(database_name: str, dump_file: str): + """Resore a database from a dump file""" + logger.info(f'Restoring the database "{database_name}" from {dump_file}') + os.system( + f'docker exec mysql mysqldump -u {MYSQL_ROOT_USER} -p{MYSQL_ROOT_PASSWORD} {database_name} < {dump_file}') + logger.info( + f"The database {database_name} was successfully retored from {dump_file}" + ) + + @ensure_mysql def reset_mysql_openedx(project: Project, dry_run: bool = False): """Run script from derex/openedx image to reset the mysql db.""" From aa8eb5613fc0345d3211dbf06754405213ea8f0f Mon Sep 17 00:00:00 2001 From: Everton Zanella Alvarenga Date: Mon, 25 Oct 2021 17:35:44 +0200 Subject: [PATCH 2/5] Adding command to dump mongodb --- derex/runner/cli/mongodb.py | 12 ++++++++++++ derex/runner/mongodb.py | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/derex/runner/cli/mongodb.py b/derex/runner/cli/mongodb.py index 8a950e1c..6374682d 100644 --- a/derex/runner/cli/mongodb.py +++ b/derex/runner/cli/mongodb.py @@ -190,3 +190,15 @@ def reset_mongodb_password_cmd(current_password: Optional[str], force: bool): reset_mongodb_password(current_password) return 0 + +@mongodb.command(name="dump") +@click.pass_obj +@click.argument("db_name", type=str) + +def dump_database_cmd(project: Optional[Project], db_name: str): + """Dump a mongodb database""" + + from derex.runner.mongodb import dump_database + + dump_database(db_name) + return 0 \ No newline at end of file diff --git a/derex/runner/mongodb.py b/derex/runner/mongodb.py index 2513b2b0..e6382210 100644 --- a/derex/runner/mongodb.py +++ b/derex/runner/mongodb.py @@ -13,6 +13,8 @@ import logging import urllib.parse +import os + logger = logging.getLogger(__name__) MONGODB_ROOT_PASSWORD = get_secret(DerexSecrets.mongodb) @@ -135,3 +137,14 @@ def reset_mongodb_password(current_password: str = None): run_ddc_services(compose_args, exit_afterwards=True) return 0 + + +@ensure_mongodb +def dump_database(database_name: str): + """Export the database""" + logger.info(f'Dumping the database "{database_name}"...') + os.system( + f'docker exec -i mongodb mongodump --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} -d {database_name} -o {database_name} && docker cp mongodb:{database_name} ./backup') + logger.info( + f"The database {database_name} was successfully dumped on {database_name}" + ) From ce90a0f9acd125ed4090da8edc201df59faabad8 Mon Sep 17 00:00:00 2001 From: Everton Zanella Alvarenga Date: Tue, 26 Oct 2021 11:41:28 +0200 Subject: [PATCH 3/5] Adding mongodb restore and dumping into a gzip file --- derex/runner/cli/mongodb.py | 11 +++++++++++ derex/runner/mongodb.py | 14 +++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/derex/runner/cli/mongodb.py b/derex/runner/cli/mongodb.py index 6374682d..13059d0c 100644 --- a/derex/runner/cli/mongodb.py +++ b/derex/runner/cli/mongodb.py @@ -201,4 +201,15 @@ def dump_database_cmd(project: Optional[Project], db_name: str): from derex.runner.mongodb import dump_database dump_database(db_name) + return 0 + +@mongodb.command(name="restore") +@click.argument("db_name", type=str, nargs=1) +@click.argument("dump_dir", type=click.Path(exists=True), nargs=1) +def restore_database_cmd(db_name: str, dump_dir: str): + """Restore a mysql database from a file""" + + from derex.runner.mongodb import restore_database + + restore_database(db_name, dump_dir) return 0 \ No newline at end of file diff --git a/derex/runner/mongodb.py b/derex/runner/mongodb.py index e6382210..d543bd65 100644 --- a/derex/runner/mongodb.py +++ b/derex/runner/mongodb.py @@ -144,7 +144,19 @@ def dump_database(database_name: str): """Export the database""" logger.info(f'Dumping the database "{database_name}"...') os.system( - f'docker exec -i mongodb mongodump --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} -d {database_name} -o {database_name} && docker cp mongodb:{database_name} ./backup') + f'docker exec -i mongodb mongodump --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} -d {database_name} --gzip --archive={database_name}.gz && docker cp mongodb:{database_name}.gz . && docker exec mongodb rm {database_name}.gz') logger.info( f"The database {database_name} was successfully dumped on {database_name}" ) + + +@ensure_mongodb +def restore_database(database_name: str, dump_file: str): + """Resore a database from a dump directory""" + logger.info( + f'Restoring the database "{database_name}" from the the directory {dump_file}') + os.system( + f'docker cp {dump_file} mongodb:/ && docker exec mongodb mongorestore --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} --db {database_name} --drop --gzip --archive={dump_file} && docker exec mongodb rm {dump_file}') + logger.info( + f"The database {database_name} was successfully retored from {dump_file}" + ) From 44c9dd05f3a3c4dfc5d2ae9ee4ad42de0098bced Mon Sep 17 00:00:00 2001 From: Everton Zanella Alvarenga Date: Tue, 26 Oct 2021 11:44:04 +0200 Subject: [PATCH 4/5] It's now a (gzip) file instead of a directory --- derex/runner/cli/mongodb.py | 6 +++--- derex/runner/mongodb.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/derex/runner/cli/mongodb.py b/derex/runner/cli/mongodb.py index 13059d0c..990c8e58 100644 --- a/derex/runner/cli/mongodb.py +++ b/derex/runner/cli/mongodb.py @@ -205,11 +205,11 @@ def dump_database_cmd(project: Optional[Project], db_name: str): @mongodb.command(name="restore") @click.argument("db_name", type=str, nargs=1) -@click.argument("dump_dir", type=click.Path(exists=True), nargs=1) -def restore_database_cmd(db_name: str, dump_dir: str): +@click.argument("dump_file", type=click.Path(exists=True), nargs=1) +def restore_database_cmd(db_name: str, dump_file: str): """Restore a mysql database from a file""" from derex.runner.mongodb import restore_database - restore_database(db_name, dump_dir) + restore_database(db_name, dump_file) return 0 \ No newline at end of file diff --git a/derex/runner/mongodb.py b/derex/runner/mongodb.py index d543bd65..ca130bc1 100644 --- a/derex/runner/mongodb.py +++ b/derex/runner/mongodb.py @@ -152,7 +152,7 @@ def dump_database(database_name: str): @ensure_mongodb def restore_database(database_name: str, dump_file: str): - """Resore a database from a dump directory""" + """Resore a database from a dump file""" logger.info( f'Restoring the database "{database_name}" from the the directory {dump_file}') os.system( From 5aa8946c91afd5762e66236ff504f5c8271d8829 Mon Sep 17 00:00:00 2001 From: Everton Zanella Alvarenga Date: Tue, 26 Oct 2021 12:15:08 +0200 Subject: [PATCH 5/5] Adding pre-commit check --- derex/runner/cli/mongodb.py | 5 +++-- derex/runner/cli/mysql.py | 1 + derex/runner/mongodb.py | 12 +++++++----- derex/runner/mysql.py | 12 ++++++------ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/derex/runner/cli/mongodb.py b/derex/runner/cli/mongodb.py index 990c8e58..b605ed77 100644 --- a/derex/runner/cli/mongodb.py +++ b/derex/runner/cli/mongodb.py @@ -191,10 +191,10 @@ def reset_mongodb_password_cmd(current_password: Optional[str], force: bool): reset_mongodb_password(current_password) return 0 + @mongodb.command(name="dump") @click.pass_obj @click.argument("db_name", type=str) - def dump_database_cmd(project: Optional[Project], db_name: str): """Dump a mongodb database""" @@ -203,6 +203,7 @@ def dump_database_cmd(project: Optional[Project], db_name: str): dump_database(db_name) return 0 + @mongodb.command(name="restore") @click.argument("db_name", type=str, nargs=1) @click.argument("dump_file", type=click.Path(exists=True), nargs=1) @@ -212,4 +213,4 @@ def restore_database_cmd(db_name: str, dump_file: str): from derex.runner.mongodb import restore_database restore_database(db_name, dump_file) - return 0 \ No newline at end of file + return 0 diff --git a/derex/runner/cli/mysql.py b/derex/runner/cli/mysql.py index 1168ad9a..fd46cd89 100644 --- a/derex/runner/cli/mysql.py +++ b/derex/runner/cli/mysql.py @@ -62,6 +62,7 @@ def drop(context: click.core.Context): def show(context: click.core.Context): """MySQL SHOW predicate""" + @create.command(name="database") @click.pass_obj @click.argument("db_name", type=str, required=False) diff --git a/derex/runner/mongodb.py b/derex/runner/mongodb.py index ca130bc1..20db5b56 100644 --- a/derex/runner/mongodb.py +++ b/derex/runner/mongodb.py @@ -11,9 +11,8 @@ from typing import Optional import logging -import urllib.parse - import os +import urllib.parse logger = logging.getLogger(__name__) @@ -144,7 +143,8 @@ def dump_database(database_name: str): """Export the database""" logger.info(f'Dumping the database "{database_name}"...') os.system( - f'docker exec -i mongodb mongodump --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} -d {database_name} --gzip --archive={database_name}.gz && docker cp mongodb:{database_name}.gz . && docker exec mongodb rm {database_name}.gz') + f"docker exec -i mongodb mongodump --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} -d {database_name} --gzip --archive={database_name}.gz && docker cp mongodb:{database_name}.gz . && docker exec mongodb rm {database_name}.gz" + ) logger.info( f"The database {database_name} was successfully dumped on {database_name}" ) @@ -154,9 +154,11 @@ def dump_database(database_name: str): def restore_database(database_name: str, dump_file: str): """Resore a database from a dump file""" logger.info( - f'Restoring the database "{database_name}" from the the directory {dump_file}') + f'Restoring the database "{database_name}" from the the directory {dump_file}' + ) os.system( - f'docker cp {dump_file} mongodb:/ && docker exec mongodb mongorestore --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} --db {database_name} --drop --gzip --archive={dump_file} && docker exec mongodb rm {dump_file}') + f"docker cp {dump_file} mongodb:/ && docker exec mongodb mongorestore --authenticationDatabase=admin -u {MONGODB_ROOT_USER} -p{MONGODB_ROOT_PASSWORD} --db {database_name} --drop --gzip --archive={dump_file} && docker exec mongodb rm {dump_file}" + ) logger.info( f"The database {database_name} was successfully retored from {dump_file}" ) diff --git a/derex/runner/mysql.py b/derex/runner/mysql.py index 331b8747..ed3fe0b0 100644 --- a/derex/runner/mysql.py +++ b/derex/runner/mysql.py @@ -14,8 +14,8 @@ from typing import Tuple import logging -import pymysql import os +import pymysql logger = logging.getLogger(__name__) @@ -189,7 +189,7 @@ def copy_database(source_db_name: str, destination_db_name: str): def dump_database(database_name: str): - """"Export the database""" + """ "Export the database""" logger.info(f'Dumping the database "{database_name}"...') # run_ddc_services( # [ @@ -200,17 +200,17 @@ def dump_database(database_name: str): # ] # ) os.system( - f'docker exec mysql mysqldump -u {MYSQL_ROOT_USER} -p{MYSQL_ROOT_PASSWORD} {database_name} > {database_name}.sql') - logger.info( - f"The database {database_name} was successfully dumped" + f"docker exec mysql mysqldump -u {MYSQL_ROOT_USER} -p{MYSQL_ROOT_PASSWORD} {database_name} > {database_name}.sql" ) + logger.info(f"The database {database_name} was successfully dumped") def restore_database(database_name: str, dump_file: str): """Resore a database from a dump file""" logger.info(f'Restoring the database "{database_name}" from {dump_file}') os.system( - f'docker exec mysql mysqldump -u {MYSQL_ROOT_USER} -p{MYSQL_ROOT_PASSWORD} {database_name} < {dump_file}') + f"docker exec mysql mysqldump -u {MYSQL_ROOT_USER} -p{MYSQL_ROOT_PASSWORD} {database_name} < {dump_file}" + ) logger.info( f"The database {database_name} was successfully retored from {dump_file}" )