From 8392a5e2112c616d8bd4022461f1bdcbd2170343 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:07:26 +0530 Subject: [PATCH 01/15] feat: set io scheduler to none --- scripts/new_setup/server_setup.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index e7b7eb0..f0e2875 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -118,15 +118,21 @@ def update_sysctl_config(): update_config("vm.vfs_cache_pressure=50", filename) -def set_io_scheduler_to_none(): +def set_io_scheduler_to_none(device_name=None): """ Set IO scheduler to none for better mariadb performance Reference: https://mariadb.com/kb/en/configuring-linux-for-mariadb/ """ - - # TODO: harddisk drive name is hardcoded - # TODO: replace existing scheduler with none. This will append. - os.system("echo none | sudo tee /sys/block/sda/queue/scheduler") + if not device_name: + # show all devices + all_devices = os.listdir("/sys/block") + print(all_devices) + while True: + device_name = input("Enter device name (e.g. sda): ") + if device_name in all_devices: + break + print("Invalid device name. Please try again.") + os.system(f"echo none | sudo tee /sys/block/{device_name}/queue/scheduler") # def create_swap_partition(): From 9dcacbab6286d41845872c748b283db26d617306 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:12:05 +0530 Subject: [PATCH 02/15] feat: create swap partation --- scripts/new_setup/config.json | 3 ++- scripts/new_setup/server_setup.py | 36 ++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/scripts/new_setup/config.json b/scripts/new_setup/config.json index 9152cc3..b89578e 100644 --- a/scripts/new_setup/config.json +++ b/scripts/new_setup/config.json @@ -4,6 +4,7 @@ "site_name": "erp.erpnext.com", "admin_password": "admin", "mariadb_root_password": "root", + "swap_size": "2", "dns_multitenant": "on", "ssh_port": 22, "dependencies": { @@ -34,4 +35,4 @@ "smit": "ssh_key", "lakshit": "lakshit_keys" } -} +} \ No newline at end of file diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index f0e2875..80e8907 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -135,16 +135,27 @@ def set_io_scheduler_to_none(device_name=None): os.system(f"echo none | sudo tee /sys/block/{device_name}/queue/scheduler") -# def create_swap_partition(): -# # check existing swap -# if os.system('swapon -s') == 0: -# return - -# # create swap partition -# os.system('sudo fallocate -l 4G /swapfile') -# os.system('sudo chmod 600 /swapfile') -# os.system('sudo mkswap /swapfile') -# os.system('sudo swapon /swapfile') +def create_swap_partition(swap_size=None): + """ + Create swap partition + Reference: https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04 + Appropriate swap size: https://help.ubuntu.com/community/SwapFaq#How_much_swap_do_I_need.3F + """ + print_step("Creating swap partition") + if not swap_size or not swap_size.isdigit(): + # show device size + os.system("free -h") + while True: + swap_size = input("Enter swap size (e.g. 1): ") + if swap_size.isdigit(): + break + print("Invalid swap size. Please try again.") + os.system(f"sudo fallocate -l {swap_size}G /swapfile") + os.system("sudo chmod 600 /swapfile") + os.system("sudo mkswap /swapfile") + os.system("sudo swapon /swapfile") + os.system("sudo cp /etc/fstab /etc/fstab.bak") + os.system("echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab") ###################################################################### @@ -325,10 +336,9 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi add_authorized_keys(username, config["authorized_keys"]) update_ssh_config(config.get("ssh_port")) update_sysctl_config() - # set_io_scheduler_to_none() - # TODO: work on this - # create_swap_partition() + set_io_scheduler_to_none() + create_swap_partition(config["swap_size"]) install_dependencies(config["dependencies"]) From 8d6a335987ee3bcf57e75ed529ca96eb5a780217 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:18:32 +0530 Subject: [PATCH 03/15] feat: install certbot and generate ssl --- scripts/new_setup/config.json | 3 ++- scripts/new_setup/server_setup.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/new_setup/config.json b/scripts/new_setup/config.json index b89578e..8268b6e 100644 --- a/scripts/new_setup/config.json +++ b/scripts/new_setup/config.json @@ -34,5 +34,6 @@ "authorized_keys": { "smit": "ssh_key", "lakshit": "lakshit_keys" - } + }, + "ssl_email": "test@gmail.com" } \ No newline at end of file diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index 80e8907..05e3d4a 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -325,6 +325,37 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi os.system(f"bench set-config dns_multitenant {dns_multitenant}") +####################################################################### +# Setup production #################################################### +####################################################################### + + +def install_certbot(): + print_step("Installing certbot") + os.system("sudo apt install snapd") + os.system("sudo snap install --classic certbot") + + +def generate_ssl_certificate(site_name, ssl_email): + print_step("Generating SSL") + os.system( + f"sudo certbot --nginx -d {site_name} -d www.{site_name} --non-interactive --agree-tos --email {ssl_email}" + ) + + +def add_ssl_to_site(site_name): + print_step("Adding SSL to site") + if not os.path.exists("/etc/letsencrypt/live/"): + print("SSL certificate not found") + return + fullchain = f"/etc/letsencrypt/live/{site_name}/fullchain.pem" + privkey = f"/etc/letsencrypt/live/{site_name}/privkey.pem" + + os.system(f"bench set-config ssl_certificate {fullchain}") + os.system(f"bench set-config ssl_certificate_key {privkey}") + os.system(f"sudo systemctl restart nginx") + + ###################################################################### From 3010f75b1801449303bdf13ccdb9f2cb76461f43 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:20:35 +0530 Subject: [PATCH 04/15] feat: setup production --- scripts/new_setup/server_setup.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index 05e3d4a..56c193f 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -330,6 +330,26 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi ####################################################################### +def setup_production(username): + print_step("Setting up production") + os.system(f"sudo bench setup production --yes {username}") + supervisorctl_permissions(username) + os.system("sudo systemctl restart supervisor") + + +def supervisorctl_permissions(username): + # add www-data to sudoers + os.system(f"sudo usermod -a -G {username} www-data") + + # add these lines under [unix_http_server] in /etc/supervisor/supervisord.conf + filepath = "/etc/supervisor/supervisord.conf" + data = f""" + chmod=0770\n + chown={username}:{username} + """ + os.system(f"sudo sed -i '6i {data}' {filepath}") + + def install_certbot(): print_step("Installing certbot") os.system("sudo apt install snapd") From 871265842da5010a45aef4fd3dee41bc342115ed Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:26:47 +0530 Subject: [PATCH 05/15] fix: removed erpnext.cnf file --- scripts/new_setup/erpnext.cnf | 64 ----------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 scripts/new_setup/erpnext.cnf diff --git a/scripts/new_setup/erpnext.cnf b/scripts/new_setup/erpnext.cnf deleted file mode 100644 index 2fb2b61..0000000 --- a/scripts/new_setup/erpnext.cnf +++ /dev/null @@ -1,64 +0,0 @@ -[mysqld] - -# GENERAL # -user = mysql -default-storage-engine = InnoDB -# socket = /var/lib/mysql/mysql.sock -# pid-file = /var/lib/mysql/mysql.pid - -# MyISAM # -key-buffer-size = 32M -myisam-recover = FORCE,BACKUP - -# SAFETY # -max-allowed-packet = 256M -max-connect-errors = 1000000 -innodb = FORCE - -# DATA STORAGE # -datadir = /var/lib/mysql/ - -# BINARY LOGGING # -log-bin = /var/lib/mysql/mysql-bin -expire-logs-days = 14 -sync-binlog = 1 - -# REPLICATION # -server-id = 1 - -# CACHES AND LIMITS # -tmp-table-size = 32M -max-heap-table-size = 32M -query-cache-type = 0 -query-cache-size = 0 -max-connections = 500 -thread-cache-size = 50 -open-files-limit = 65535 -table-definition-cache = 4096 -table-open-cache = 10240 - -# INNODB # -innodb-flush-method = O_DIRECT -innodb-log-files-in-group = 2 -innodb-log-file-size = 512M -innodb-flush-log-at-trx-commit = 1 -innodb-file-per-table = 1 -innodb-buffer-pool-size = 2640M -innodb-file-format = barracuda -innodb-large-prefix = 1 -collation-server = utf8mb4_unicode_ci -character-set-server = utf8mb4 -character-set-client-handshake = FALSE -max_allowed_packet = 256M - -# LOGGING # -log-error = /var/lib/mysql/mysql-error.log -log-queries-not-using-indexes = 0 -slow-query-log = 1 -slow-query-log-file = /var/lib/mysql/mysql-slow.log - -[mysql] -default-character-set = utf8mb4 - -[mysqldump] -max_allowed_packet=256M \ No newline at end of file From 9bceda7053c08514cadb409ef2d9a43601deb68a Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:27:47 +0530 Subject: [PATCH 06/15] fix: create maridb cnf file with dynamic innodb buffer pool size --- scripts/new_setup/config.json | 1 + scripts/new_setup/server_setup.py | 104 ++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/scripts/new_setup/config.json b/scripts/new_setup/config.json index 8268b6e..504e5f3 100644 --- a/scripts/new_setup/config.json +++ b/scripts/new_setup/config.json @@ -5,6 +5,7 @@ "admin_password": "admin", "mariadb_root_password": "root", "swap_size": "2", + "innodb_buffer_pool_size": "0", "dns_multitenant": "on", "ssh_port": 22, "dependencies": { diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index 56c193f..c7d776d 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -199,7 +199,7 @@ def mysql_secure_installation(): os.system("sudo mysql_secure_installation") -def update_mariadb_config(): +def update_mariadb_config(innodb_buffer_pool_size=None): """ Create config files: 1. /etc/mysql/mariadb.conf.d/erpnext.cnf @@ -207,11 +207,8 @@ def update_mariadb_config(): """ print_step("Updating mariadb config") - # move erpnext.cnf file - dir = "/etc/mysql/mariadb.conf.d" - filename = "erpnext.cnf" - os.system(f"sudo mkdir -p {dir}") - os.system(f"sudo cp erpnext.cnf {os.path.join(dir, filename)}") + # update erpnext.cnf + create_erpnect_cnf(innodb_buffer_pool_size) # update override.conf dir = "/etc/systemd/system/mariadb.service.d" @@ -376,6 +373,101 @@ def add_ssl_to_site(site_name): os.system(f"sudo systemctl restart nginx") +####################################################################### +# erpnext.cnf############################################################ +####################################################################### + + +def create_erpnect_cnf(innodb_buffer_pool_size=None): + # create erpnext.cnf file + print_step("Creating erpnext.cnf file") + # create file + dir = "/etc/mysql/mariadb.conf.d" + filename = "erpnext.cnf" + os.system(f"sudo mkdir -p {dir}") + os.system(f"sudo touch {dir}/{filename}") + file_path = f"{dir}/{filename}" + # calculate innodb_buffer_pool_size + if innodb_buffer_pool_size is None or innodb_buffer_pool_size == 0: + innodb_buffer_pool_size = calculate_innodb_buffer_pool_size() + data = f""" + [mysqld] + + # GENERAL # + user = mysql + default-storage-engine = InnoDB + # socket = /var/lib/mysql/mysql.sock + # pid-file = /var/lib/mysql/mysql.pid + + # MyISAM # + key-buffer-size = 32M + myisam-recover = FORCE,BACKUP + + # SAFETY # + max-allowed-packet = 256M + max-connect-errors = 1000000 + innodb = FORCE + + # DATA STORAGE # + datadir = /var/lib/mysql/ + + # BINARY LOGGING # + log-bin = /var/lib/mysql/mysql-bin + expire-logs-days = 14 + sync-binlog = 1 + + # REPLICATION # + server-id = 1 + + # CACHES AND LIMITS # + tmp-table-size = 32M + max-heap-table-size = 32M + query-cache-type = 0 + query-cache-size = 0 + max-connections = 500 + thread-cache-size = 50 + open-files-limit = 65535 + table-definition-cache = 4096 + table-open-cache = 10240 + + # INNODB # + innodb-flush-method = O_DIRECT + innodb-log-files-in-group = 2 + innodb-log-file-size = 512M + innodb-flush-log-at-trx-commit = 1 + innodb-file-per-table = 1 + innodb-buffer-pool-size = {innodb_buffer_pool_size}M + innodb-file-format = barracuda + innodb-large-prefix = 1 + collation-server = utf8mb4_unicode_ci + character-set-server = utf8mb4 + character-set-client-handshake = FALSE + max_allowed_packet = 256M + + # LOGGING # + log-error = /var/lib/mysql/mysql-error.log + log-queries-not-using-indexes = 0 + slow-query-log = 1 + slow-query-log-file = /var/lib/mysql/mysql-slow.log + + [mysql] + default-character-set = utf8mb4 + + [mysqldump] + max_allowed_packet=256M + """ + update_config(data, file_path) + + +def calculate_innodb_buffer_pool_size(): + # calculate innodb buffer pool size + print_step("Calculating innodb buffer pool size") + free_memory = os.popen("free -m | awk '/^Mem:/{print $4}'").read() + innodb_buffer_pool_size = int(free_memory * 0.6) + # convert value to whole number + return innodb_buffer_pool_size + + ###################################################################### From 386ce71ee932d323e61a58db6ff8af89ebdfad83 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:39:06 +0530 Subject: [PATCH 07/15] fix: change dir to bench to run bench command --- scripts/new_setup/server_setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index c7d776d..f8bc1f9 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -327,8 +327,9 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi ####################################################################### -def setup_production(username): +def setup_production(username, bench_name): print_step("Setting up production") + os.chdir(f"/home/{username}/{bench_name}") os.system(f"sudo bench setup production --yes {username}") supervisorctl_permissions(username) os.system("sudo systemctl restart supervisor") @@ -360,7 +361,7 @@ def generate_ssl_certificate(site_name, ssl_email): ) -def add_ssl_to_site(site_name): +def add_ssl_to_site(site_name, username, bench_name): print_step("Adding SSL to site") if not os.path.exists("/etc/letsencrypt/live/"): print("SSL certificate not found") @@ -368,6 +369,7 @@ def add_ssl_to_site(site_name): fullchain = f"/etc/letsencrypt/live/{site_name}/fullchain.pem" privkey = f"/etc/letsencrypt/live/{site_name}/privkey.pem" + os.chdir(f"/home/{username}/{bench_name}") os.system(f"bench set-config ssl_certificate {fullchain}") os.system(f"bench set-config ssl_certificate_key {privkey}") os.system(f"sudo systemctl restart nginx") From 30862c9aabab0f94d4ee3430aeaed190cab9a4d5 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:54:15 +0530 Subject: [PATCH 08/15] fix created functions for individual process --- scripts/new_setup/server_setup.py | 64 +++++++++++++++++++------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/server_setup.py index f8bc1f9..21c7b27 100644 --- a/scripts/new_setup/server_setup.py +++ b/scripts/new_setup/server_setup.py @@ -327,9 +327,8 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi ####################################################################### -def setup_production(username, bench_name): +def setup_production(username): print_step("Setting up production") - os.chdir(f"/home/{username}/{bench_name}") os.system(f"sudo bench setup production --yes {username}") supervisorctl_permissions(username) os.system("sudo systemctl restart supervisor") @@ -361,7 +360,7 @@ def generate_ssl_certificate(site_name, ssl_email): ) -def add_ssl_to_site(site_name, username, bench_name): +def add_ssl_to_site(site_name): print_step("Adding SSL to site") if not os.path.exists("/etc/letsencrypt/live/"): print("SSL certificate not found") @@ -369,7 +368,6 @@ def add_ssl_to_site(site_name, username, bench_name): fullchain = f"/etc/letsencrypt/live/{site_name}/fullchain.pem" privkey = f"/etc/letsencrypt/live/{site_name}/privkey.pem" - os.chdir(f"/home/{username}/{bench_name}") os.system(f"bench set-config ssl_certificate {fullchain}") os.system(f"bench set-config ssl_certificate_key {privkey}") os.system(f"sudo systemctl restart nginx") @@ -471,33 +469,49 @@ def calculate_innodb_buffer_pool_size(): ###################################################################### +# Tasks############################################################### +###################################################################### -if __name__ == "__main__": - config = read_server_script_json() - username = config["username"] - +def update_server_config(username, keys, ssh_keys): + print_step("Updating server config") update_and_upgrade_apt() - add_authorized_keys(username, config["authorized_keys"]) - update_ssh_config(config.get("ssh_port")) - update_sysctl_config() + add_authorized_keys(username, keys) + update_ssh_config(ssh_keys) + +def update_system_for_mariadb(swap_size): + print_step("Updating system for mariadb") + update_sysctl_config() set_io_scheduler_to_none() - create_swap_partition(config["swap_size"]) + create_swap_partition(swap_size) - install_dependencies(config["dependencies"]) - # init frappe-bench +def install_bench(dependencies): + print_step("Installing bench") + install_dependencies(dependencies) install_frappe_bench() - intialize_frappe_bench( - username, config["dependencies"]["python"], config["apps"], config["bench_name"] - ) - setup_site( - config["site_name"], - config["mariadb_root_password"], - config["admin_password"], - config["apps"], - config["dns_multitenant"], - ) - os.system("exit") + +def intialize_bench_with_apps(username, version, apps, bench_name): + print_step("Intializing bench with apps") + intialize_frappe_bench(username, version, apps, bench_name) + + +def create_site_with_app( + site_name, + mariadb_root_password, + admin_password, + apps, + dns_multitenant, + username, + bench_name, +): + print_step("Creating site with app") + os.chdir(f"/home/{username}/{bench_name}") + setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multitenant) + + +def setup_production_server(username, bench_name): + os.chdir(f"/home/{username}/{bench_name}") + setup_production(username) From 6d99a11893de8ccb2000470f5f20a09bdcfc8611 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 16:56:06 +0530 Subject: [PATCH 09/15] fix: file renamed --- scripts/new_setup/{server_setup.py => utils.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/new_setup/{server_setup.py => utils.py} (100%) diff --git a/scripts/new_setup/server_setup.py b/scripts/new_setup/utils.py similarity index 100% rename from scripts/new_setup/server_setup.py rename to scripts/new_setup/utils.py From 9a0dc6800ad1a46aa26ab9072184e1910e79096e Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 17:12:38 +0530 Subject: [PATCH 10/15] fix created main.py to select functions --- scripts/new_setup/main.py | 60 +++++++++++++++++++++++++ scripts/new_setup/utils.py | 92 +++++++++++++++----------------------- 2 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 scripts/new_setup/main.py diff --git a/scripts/new_setup/main.py b/scripts/new_setup/main.py new file mode 100644 index 0000000..1946802 --- /dev/null +++ b/scripts/new_setup/main.py @@ -0,0 +1,60 @@ +# main file for task selection +from utils import * + +""" +This file will be executed on the server after the server is created. + +Requirements: +============= +1. Python 3.6 or above +2. Git +3. Tested on Ubuntu 22.04 + +Steps for Running This Script: +============================== +1. Create a new user on the server (make sure its same as the username in config.json) +2. Add the user to sudoers +3. Login as the new user +4. Install git +5. Clone this repo +6. Go to the repo directory +7. Run this script after updating the config.json file + +eg for a new user named `frappe` +```sh +sudo adduser frappe +sudo usermod -aG sudo frappe +su - frappe + +sudo apt-get update && sudo apt-get install git -y +git clone https://github.com/vorasmit/installer.git +cd installer/scripts/new_setup +python3 main.py +``` + +""" +if __name__ == "__main__": + while True: + print("Select Task", end="\n\n\n") + task_list = { + "Update Server Config": update_server_config, + "Update System for MariaDB": update_system_for_mariadb, + "Install Bench": install_bench, + "Initialize Bench with Apps": intialize_bench_with_apps, + "Create Site with App": create_site_with_app, + "Setup SSL": setup_ssl, + "Setup Production": setup_production_server, + } + + for key, value in enumerate(task_list.items()): + print(f"{key + 1}. {value[0]}") + + print("Enter 'q' to quit") + task = input("\n\nEnter the task number: ") + + try: + if task == "q": + break + task_list[list(task_list.keys())[int(task) - 1]]() + except Exception as e: + print(e) diff --git a/scripts/new_setup/utils.py b/scripts/new_setup/utils.py index 21c7b27..55d9bbd 100644 --- a/scripts/new_setup/utils.py +++ b/scripts/new_setup/utils.py @@ -1,38 +1,4 @@ #!/usr/bin/env python - -""" -This file will be executed on the server after the server is created. - -Requirements: -============= -1. Python 3.6 or above -2. Git -3. Tested on Ubuntu 22.04 - -Steps for Running This Script: -============================== -1. Create a new user on the server (make sure its same as the username in config.json) -2. Add the user to sudoers -3. Login as the new user -4. Install git -5. Clone this repo -6. Go to the repo directory -7. Run this script after updating the config.json file - -eg for a new user named `frappe` -```sh -sudo adduser frappe -sudo usermod -aG sudo frappe -su - frappe - -sudo apt-get update && sudo apt-get install git -y -git clone https://github.com/vorasmit/installer.git -cd installer/scripts/new_setup -python3 server_script.py -``` - -""" - import os import json @@ -181,7 +147,7 @@ def install_python(version): ) -def install_mariadb(version): +def install_mariadb(version, innodb_buffer_pool_size=None): print_step("Installing mariadb " + version) os.system("wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup") os.system("chmod +x mariadb_repo_setup") @@ -192,7 +158,7 @@ def install_mariadb(version): os.system("rm mariadb_repo_setup") mysql_secure_installation() - update_mariadb_config() + update_mariadb_config(innodb_buffer_pool_size) def mysql_secure_installation(): @@ -472,46 +438,58 @@ def calculate_innodb_buffer_pool_size(): # Tasks############################################################### ###################################################################### +config = read_server_script_json() +username = config["username"] +bench_name = config["bench_name"] + -def update_server_config(username, keys, ssh_keys): +def update_server_config(): print_step("Updating server config") update_and_upgrade_apt() - add_authorized_keys(username, keys) - update_ssh_config(ssh_keys) + add_authorized_keys(username, config["authorized_keys"]) + update_ssh_config(config.get("ssh_port")) -def update_system_for_mariadb(swap_size): +def update_system_for_mariadb(): print_step("Updating system for mariadb") update_sysctl_config() set_io_scheduler_to_none() - create_swap_partition(swap_size) + create_swap_partition(config["swap_size"]) -def install_bench(dependencies): +def install_bench(): print_step("Installing bench") - install_dependencies(dependencies) + install_dependencies() install_frappe_bench() -def intialize_bench_with_apps(username, version, apps, bench_name): +def intialize_bench_with_apps(): print_step("Intializing bench with apps") - intialize_frappe_bench(username, version, apps, bench_name) - - -def create_site_with_app( - site_name, - mariadb_root_password, - admin_password, - apps, - dns_multitenant, - username, - bench_name, -): + intialize_frappe_bench( + username, config["dependencies"]["python"], config["apps"], config["bench_name"] + ) + + +def create_site_with_app(): print_step("Creating site with app") os.chdir(f"/home/{username}/{bench_name}") - setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multitenant) + setup_site( + config["site_name"], + config["mariadb_root_password"], + config["admin_password"], + config["apps"], + config["dns_multitenant"], + ) def setup_production_server(username, bench_name): os.chdir(f"/home/{username}/{bench_name}") setup_production(username) + + +def setup_ssl(): + print_step("Generate SSL") + install_certbot() + generate_ssl_certificate(config["site_name"], config["ssl_email"]) + os.chdir(f"/home/{username}/{bench_name}") + add_ssl_to_site(config["site_name"]) From cb0bcc4be9abc76538d448e407c74e6c714d0ab9 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 17:28:34 +0530 Subject: [PATCH 11/15] fix: minor fixes on review --- scripts/new_setup/utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/new_setup/utils.py b/scripts/new_setup/utils.py index 55d9bbd..6788bd9 100644 --- a/scripts/new_setup/utils.py +++ b/scripts/new_setup/utils.py @@ -92,7 +92,9 @@ def set_io_scheduler_to_none(device_name=None): if not device_name: # show all devices all_devices = os.listdir("/sys/block") - print(all_devices) + print_step("Available devices") + print(all_devices, end="\n\n") + while True: device_name = input("Enter device name (e.g. sda): ") if device_name in all_devices: @@ -130,6 +132,7 @@ def create_swap_partition(swap_size=None): def install_dependencies(dependencies): + print_step("Installing dependencies") install_python(dependencies["python"]) install_mariadb(dependencies["mariadb"]) install_nodejs(dependencies["node"]) @@ -429,7 +432,8 @@ def calculate_innodb_buffer_pool_size(): # calculate innodb buffer pool size print_step("Calculating innodb buffer pool size") free_memory = os.popen("free -m | awk '/^Mem:/{print $4}'").read() - innodb_buffer_pool_size = int(free_memory * 0.6) + innodb_buffer_pool_size = int(int(free_memory) * 0.6) + print(f"innodb_buffer_pool_size: {innodb_buffer_pool_size}M") # convert value to whole number return innodb_buffer_pool_size @@ -459,7 +463,7 @@ def update_system_for_mariadb(): def install_bench(): print_step("Installing bench") - install_dependencies() + install_dependencies(config["dependencies"]) install_frappe_bench() From c7c8f74c6b205af06662bd0351c4782d4d1f4052 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 17:42:24 +0530 Subject: [PATCH 12/15] fix: updated calculation value for innodb buffer pool size --- scripts/new_setup/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/new_setup/utils.py b/scripts/new_setup/utils.py index 6788bd9..2bebe92 100644 --- a/scripts/new_setup/utils.py +++ b/scripts/new_setup/utils.py @@ -432,7 +432,7 @@ def calculate_innodb_buffer_pool_size(): # calculate innodb buffer pool size print_step("Calculating innodb buffer pool size") free_memory = os.popen("free -m | awk '/^Mem:/{print $4}'").read() - innodb_buffer_pool_size = int(int(free_memory) * 0.6) + innodb_buffer_pool_size = int(int(free_memory) * 0.685) print(f"innodb_buffer_pool_size: {innodb_buffer_pool_size}M") # convert value to whole number return innodb_buffer_pool_size From db2d527dfde31d09f5958d5993c64d684435bd18 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 17:45:30 +0530 Subject: [PATCH 13/15] chore: files relocated --- scripts/new_setup/config.json => config.json | 0 scripts/new_setup/main.py => main.py | 0 scripts/new_setup/utils.py => utils.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename scripts/new_setup/config.json => config.json (100%) rename scripts/new_setup/main.py => main.py (100%) rename scripts/new_setup/utils.py => utils.py (100%) diff --git a/scripts/new_setup/config.json b/config.json similarity index 100% rename from scripts/new_setup/config.json rename to config.json diff --git a/scripts/new_setup/main.py b/main.py similarity index 100% rename from scripts/new_setup/main.py rename to main.py diff --git a/scripts/new_setup/utils.py b/utils.py similarity index 100% rename from scripts/new_setup/utils.py rename to utils.py From 8a814f7a27695084a52c463f88441a7125c4794e Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 17:58:14 +0530 Subject: [PATCH 14/15] fix: minor fix with refactor --- main.py | 2 +- utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 1946802..4bc2a55 100644 --- a/main.py +++ b/main.py @@ -42,8 +42,8 @@ "Install Bench": install_bench, "Initialize Bench with Apps": intialize_bench_with_apps, "Create Site with App": create_site_with_app, - "Setup SSL": setup_ssl, "Setup Production": setup_production_server, + "Setup SSL": setup_ssl, } for key, value in enumerate(task_list.items()): diff --git a/utils.py b/utils.py index 2bebe92..598b8c7 100644 --- a/utils.py +++ b/utils.py @@ -486,7 +486,7 @@ def create_site_with_app(): ) -def setup_production_server(username, bench_name): +def setup_production_server(): os.chdir(f"/home/{username}/{bench_name}") setup_production(username) From 66cbdfb82287b587293a082d1ca9a381f17c0249 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Wed, 15 Mar 2023 18:24:59 +0530 Subject: [PATCH 15/15] fix: updating supervisor.conf --- utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/utils.py b/utils.py index 598b8c7..c07ec2e 100644 --- a/utils.py +++ b/utils.py @@ -309,11 +309,9 @@ def supervisorctl_permissions(username): # add these lines under [unix_http_server] in /etc/supervisor/supervisord.conf filepath = "/etc/supervisor/supervisord.conf" - data = f""" - chmod=0770\n - chown={username}:{username} - """ - os.system(f"sudo sed -i '6i {data}' {filepath}") + + os.system(f"""sudo sed -i '6i chmod=0760' {filepath}""") + os.system(f"""sudo sed -i '7i chown={username}:{username}' {filepath}""") def install_certbot(): @@ -342,6 +340,11 @@ def add_ssl_to_site(site_name): os.system(f"sudo systemctl restart nginx") +def setup_supervisor(): + print_step("Setting up supervisor") + os.system("sudo bench setup supervisor") + + ####################################################################### # erpnext.cnf############################################################ #######################################################################