diff --git a/scripts/new_setup/config.json b/config.json similarity index 90% rename from scripts/new_setup/config.json rename to config.json index 9152cc3..504e5f3 100644 --- a/scripts/new_setup/config.json +++ b/config.json @@ -4,6 +4,8 @@ "site_name": "erp.erpnext.com", "admin_password": "admin", "mariadb_root_password": "root", + "swap_size": "2", + "innodb_buffer_pool_size": "0", "dns_multitenant": "on", "ssh_port": 22, "dependencies": { @@ -33,5 +35,6 @@ "authorized_keys": { "smit": "ssh_key", "lakshit": "lakshit_keys" - } -} + }, + "ssl_email": "test@gmail.com" +} \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..4bc2a55 --- /dev/null +++ b/main.py @@ -0,0 +1,60 @@ +# main file for task selection +from utils import * + +""" +This file will be executed on the server after the server is created. + +Requirements: +============= +1. Python 3.6 or above +2. Git +3. Tested on Ubuntu 22.04 + +Steps for Running This Script: +============================== +1. Create a new user on the server (make sure its same as the username in config.json) +2. Add the user to sudoers +3. Login as the new user +4. Install git +5. Clone this repo +6. Go to the repo directory +7. Run this script after updating the config.json file + +eg for a new user named `frappe` +```sh +sudo adduser frappe +sudo usermod -aG sudo frappe +su - frappe + +sudo apt-get update && sudo apt-get install git -y +git clone https://github.com/vorasmit/installer.git +cd installer/scripts/new_setup +python3 main.py +``` + +""" +if __name__ == "__main__": + while True: + print("Select Task", end="\n\n\n") + task_list = { + "Update Server Config": update_server_config, + "Update System for MariaDB": update_system_for_mariadb, + "Install Bench": install_bench, + "Initialize Bench with Apps": intialize_bench_with_apps, + "Create Site with App": create_site_with_app, + "Setup Production": setup_production_server, + "Setup SSL": setup_ssl, + } + + for key, value in enumerate(task_list.items()): + print(f"{key + 1}. {value[0]}") + + print("Enter 'q' to quit") + task = input("\n\nEnter the task number: ") + + try: + if task == "q": + break + task_list[list(task_list.keys())[int(task) - 1]]() + except Exception as e: + print(e) diff --git a/scripts/new_setup/erpnext.cnf b/scripts/new_setup/erpnext.cnf deleted file mode 100644 index 2fb2b61..0000000 --- a/scripts/new_setup/erpnext.cnf +++ /dev/null @@ -1,64 +0,0 @@ -[mysqld] - -# GENERAL # -user = mysql -default-storage-engine = InnoDB -# socket = /var/lib/mysql/mysql.sock -# pid-file = /var/lib/mysql/mysql.pid - -# MyISAM # -key-buffer-size = 32M -myisam-recover = FORCE,BACKUP - -# SAFETY # -max-allowed-packet = 256M -max-connect-errors = 1000000 -innodb = FORCE - -# DATA STORAGE # -datadir = /var/lib/mysql/ - -# BINARY LOGGING # -log-bin = /var/lib/mysql/mysql-bin -expire-logs-days = 14 -sync-binlog = 1 - -# REPLICATION # -server-id = 1 - -# CACHES AND LIMITS # -tmp-table-size = 32M -max-heap-table-size = 32M -query-cache-type = 0 -query-cache-size = 0 -max-connections = 500 -thread-cache-size = 50 -open-files-limit = 65535 -table-definition-cache = 4096 -table-open-cache = 10240 - -# INNODB # -innodb-flush-method = O_DIRECT -innodb-log-files-in-group = 2 -innodb-log-file-size = 512M -innodb-flush-log-at-trx-commit = 1 -innodb-file-per-table = 1 -innodb-buffer-pool-size = 2640M -innodb-file-format = barracuda -innodb-large-prefix = 1 -collation-server = utf8mb4_unicode_ci -character-set-server = utf8mb4 -character-set-client-handshake = FALSE -max_allowed_packet = 256M - -# LOGGING # -log-error = /var/lib/mysql/mysql-error.log -log-queries-not-using-indexes = 0 -slow-query-log = 1 -slow-query-log-file = /var/lib/mysql/mysql-slow.log - -[mysql] -default-character-set = utf8mb4 - -[mysqldump] -max_allowed_packet=256M \ No newline at end of file diff --git a/scripts/new_setup/server_setup.py b/utils.py similarity index 50% rename from scripts/new_setup/server_setup.py rename to utils.py index e7b7eb0..c07ec2e 100644 --- a/scripts/new_setup/server_setup.py +++ b/utils.py @@ -1,38 +1,4 @@ #!/usr/bin/env python - -""" -This file will be executed on the server after the server is created. - -Requirements: -============= -1. Python 3.6 or above -2. Git -3. Tested on Ubuntu 22.04 - -Steps for Running This Script: -============================== -1. Create a new user on the server (make sure its same as the username in config.json) -2. Add the user to sudoers -3. Login as the new user -4. Install git -5. Clone this repo -6. Go to the repo directory -7. Run this script after updating the config.json file - -eg for a new user named `frappe` -```sh -sudo adduser frappe -sudo usermod -aG sudo frappe -su - frappe - -sudo apt-get update && sudo apt-get install git -y -git clone https://github.com/vorasmit/installer.git -cd installer/scripts/new_setup -python3 server_script.py -``` - -""" - import os import json @@ -118,27 +84,46 @@ def update_sysctl_config(): update_config("vm.vfs_cache_pressure=50", filename) -def set_io_scheduler_to_none(): +def set_io_scheduler_to_none(device_name=None): """ Set IO scheduler to none for better mariadb performance Reference: https://mariadb.com/kb/en/configuring-linux-for-mariadb/ """ + if not device_name: + # show all devices + all_devices = os.listdir("/sys/block") + print_step("Available devices") + print(all_devices, end="\n\n") - # TODO: harddisk drive name is hardcoded - # TODO: replace existing scheduler with none. This will append. - os.system("echo none | sudo tee /sys/block/sda/queue/scheduler") + while True: + device_name = input("Enter device name (e.g. sda): ") + if device_name in all_devices: + break + print("Invalid device name. Please try again.") + os.system(f"echo none | sudo tee /sys/block/{device_name}/queue/scheduler") -# def create_swap_partition(): -# # check existing swap -# if os.system('swapon -s') == 0: -# return - -# # create swap partition -# os.system('sudo fallocate -l 4G /swapfile') -# os.system('sudo chmod 600 /swapfile') -# os.system('sudo mkswap /swapfile') -# os.system('sudo swapon /swapfile') +def create_swap_partition(swap_size=None): + """ + Create swap partition + Reference: https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-22-04 + Appropriate swap size: https://help.ubuntu.com/community/SwapFaq#How_much_swap_do_I_need.3F + """ + print_step("Creating swap partition") + if not swap_size or not swap_size.isdigit(): + # show device size + os.system("free -h") + while True: + swap_size = input("Enter swap size (e.g. 1): ") + if swap_size.isdigit(): + break + print("Invalid swap size. Please try again.") + os.system(f"sudo fallocate -l {swap_size}G /swapfile") + os.system("sudo chmod 600 /swapfile") + os.system("sudo mkswap /swapfile") + os.system("sudo swapon /swapfile") + os.system("sudo cp /etc/fstab /etc/fstab.bak") + os.system("echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab") ###################################################################### @@ -147,6 +132,7 @@ def set_io_scheduler_to_none(): def install_dependencies(dependencies): + print_step("Installing dependencies") install_python(dependencies["python"]) install_mariadb(dependencies["mariadb"]) install_nodejs(dependencies["node"]) @@ -164,7 +150,7 @@ def install_python(version): ) -def install_mariadb(version): +def install_mariadb(version, innodb_buffer_pool_size=None): print_step("Installing mariadb " + version) os.system("wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup") os.system("chmod +x mariadb_repo_setup") @@ -175,14 +161,14 @@ def install_mariadb(version): os.system("rm mariadb_repo_setup") mysql_secure_installation() - update_mariadb_config() + update_mariadb_config(innodb_buffer_pool_size) def mysql_secure_installation(): os.system("sudo mysql_secure_installation") -def update_mariadb_config(): +def update_mariadb_config(innodb_buffer_pool_size=None): """ Create config files: 1. /etc/mysql/mariadb.conf.d/erpnext.cnf @@ -190,11 +176,8 @@ def update_mariadb_config(): """ print_step("Updating mariadb config") - # move erpnext.cnf file - dir = "/etc/mysql/mariadb.conf.d" - filename = "erpnext.cnf" - os.system(f"sudo mkdir -p {dir}") - os.system(f"sudo cp erpnext.cnf {os.path.join(dir, filename)}") + # update erpnext.cnf + create_erpnect_cnf(innodb_buffer_pool_size) # update override.conf dir = "/etc/systemd/system/mariadb.service.d" @@ -308,29 +291,195 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi os.system(f"bench set-config dns_multitenant {dns_multitenant}") +####################################################################### +# Setup production #################################################### +####################################################################### + + +def setup_production(username): + print_step("Setting up production") + os.system(f"sudo bench setup production --yes {username}") + supervisorctl_permissions(username) + os.system("sudo systemctl restart supervisor") + + +def supervisorctl_permissions(username): + # add www-data to sudoers + os.system(f"sudo usermod -a -G {username} www-data") + + # add these lines under [unix_http_server] in /etc/supervisor/supervisord.conf + filepath = "/etc/supervisor/supervisord.conf" + + os.system(f"""sudo sed -i '6i chmod=0760' {filepath}""") + os.system(f"""sudo sed -i '7i chown={username}:{username}' {filepath}""") + + +def install_certbot(): + print_step("Installing certbot") + os.system("sudo apt install snapd") + os.system("sudo snap install --classic certbot") + + +def generate_ssl_certificate(site_name, ssl_email): + print_step("Generating SSL") + os.system( + f"sudo certbot --nginx -d {site_name} -d www.{site_name} --non-interactive --agree-tos --email {ssl_email}" + ) + + +def add_ssl_to_site(site_name): + print_step("Adding SSL to site") + if not os.path.exists("/etc/letsencrypt/live/"): + print("SSL certificate not found") + return + fullchain = f"/etc/letsencrypt/live/{site_name}/fullchain.pem" + privkey = f"/etc/letsencrypt/live/{site_name}/privkey.pem" + + os.system(f"bench set-config ssl_certificate {fullchain}") + os.system(f"bench set-config ssl_certificate_key {privkey}") + os.system(f"sudo systemctl restart nginx") + + +def setup_supervisor(): + print_step("Setting up supervisor") + os.system("sudo bench setup supervisor") + + +####################################################################### +# erpnext.cnf############################################################ +####################################################################### + + +def create_erpnect_cnf(innodb_buffer_pool_size=None): + # create erpnext.cnf file + print_step("Creating erpnext.cnf file") + # create file + dir = "/etc/mysql/mariadb.conf.d" + filename = "erpnext.cnf" + os.system(f"sudo mkdir -p {dir}") + os.system(f"sudo touch {dir}/{filename}") + file_path = f"{dir}/{filename}" + # calculate innodb_buffer_pool_size + if innodb_buffer_pool_size is None or innodb_buffer_pool_size == 0: + innodb_buffer_pool_size = calculate_innodb_buffer_pool_size() + data = f""" + [mysqld] + + # GENERAL # + user = mysql + default-storage-engine = InnoDB + # socket = /var/lib/mysql/mysql.sock + # pid-file = /var/lib/mysql/mysql.pid + + # MyISAM # + key-buffer-size = 32M + myisam-recover = FORCE,BACKUP + + # SAFETY # + max-allowed-packet = 256M + max-connect-errors = 1000000 + innodb = FORCE + + # DATA STORAGE # + datadir = /var/lib/mysql/ + + # BINARY LOGGING # + log-bin = /var/lib/mysql/mysql-bin + expire-logs-days = 14 + sync-binlog = 1 + + # REPLICATION # + server-id = 1 + + # CACHES AND LIMITS # + tmp-table-size = 32M + max-heap-table-size = 32M + query-cache-type = 0 + query-cache-size = 0 + max-connections = 500 + thread-cache-size = 50 + open-files-limit = 65535 + table-definition-cache = 4096 + table-open-cache = 10240 + + # INNODB # + innodb-flush-method = O_DIRECT + innodb-log-files-in-group = 2 + innodb-log-file-size = 512M + innodb-flush-log-at-trx-commit = 1 + innodb-file-per-table = 1 + innodb-buffer-pool-size = {innodb_buffer_pool_size}M + innodb-file-format = barracuda + innodb-large-prefix = 1 + collation-server = utf8mb4_unicode_ci + character-set-server = utf8mb4 + character-set-client-handshake = FALSE + max_allowed_packet = 256M + + # LOGGING # + log-error = /var/lib/mysql/mysql-error.log + log-queries-not-using-indexes = 0 + slow-query-log = 1 + slow-query-log-file = /var/lib/mysql/mysql-slow.log + + [mysql] + default-character-set = utf8mb4 + + [mysqldump] + max_allowed_packet=256M + """ + update_config(data, file_path) + + +def calculate_innodb_buffer_pool_size(): + # calculate innodb buffer pool size + print_step("Calculating innodb buffer pool size") + free_memory = os.popen("free -m | awk '/^Mem:/{print $4}'").read() + innodb_buffer_pool_size = int(int(free_memory) * 0.685) + print(f"innodb_buffer_pool_size: {innodb_buffer_pool_size}M") + # convert value to whole number + return innodb_buffer_pool_size + + +###################################################################### +# Tasks############################################################### ###################################################################### +config = read_server_script_json() +username = config["username"] +bench_name = config["bench_name"] -if __name__ == "__main__": - config = read_server_script_json() - username = config["username"] +def update_server_config(): + print_step("Updating server config") update_and_upgrade_apt() add_authorized_keys(username, config["authorized_keys"]) update_ssh_config(config.get("ssh_port")) + + +def update_system_for_mariadb(): + print_step("Updating system for mariadb") update_sysctl_config() - # set_io_scheduler_to_none() + set_io_scheduler_to_none() + create_swap_partition(config["swap_size"]) - # TODO: work on this - # create_swap_partition() +def install_bench(): + print_step("Installing bench") install_dependencies(config["dependencies"]) - - # init frappe-bench install_frappe_bench() + + +def intialize_bench_with_apps(): + print_step("Intializing bench with apps") intialize_frappe_bench( username, config["dependencies"]["python"], config["apps"], config["bench_name"] ) + + +def create_site_with_app(): + print_step("Creating site with app") + os.chdir(f"/home/{username}/{bench_name}") setup_site( config["site_name"], config["mariadb_root_password"], @@ -339,4 +488,15 @@ def setup_site(site_name, mariadb_root_password, admin_password, apps, dns_multi config["dns_multitenant"], ) - os.system("exit") + +def setup_production_server(): + os.chdir(f"/home/{username}/{bench_name}") + setup_production(username) + + +def setup_ssl(): + print_step("Generate SSL") + install_certbot() + generate_ssl_certificate(config["site_name"], config["ssl_email"]) + os.chdir(f"/home/{username}/{bench_name}") + add_ssl_to_site(config["site_name"])