From f1dead86582b036d33e6eb3ab0b325320772c44e Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 2 Oct 2024 12:43:48 +0200 Subject: [PATCH 1/8] Added usergroup error checker --- .../check_usergroups_error.py | 102 ++++++++++++++++++ .../example_config.json | 9 ++ 2 files changed, 111 insertions(+) create mode 100644 DHIS2/usergroups_error_notificator/check_usergroups_error.py create mode 100644 DHIS2/usergroups_error_notificator/example_config.json diff --git a/DHIS2/usergroups_error_notificator/check_usergroups_error.py b/DHIS2/usergroups_error_notificator/check_usergroups_error.py new file mode 100644 index 00000000..45e862e1 --- /dev/null +++ b/DHIS2/usergroups_error_notificator/check_usergroups_error.py @@ -0,0 +1,102 @@ +import json +from datetime import datetime +import requests +from requests.auth import HTTPBasicAuth +import argparse + +changes_detected = False +server = "" +auth = "" +all_users_request = "/api/users.json?fields=id&paging=false" + + +def get_user_groups(user_id): + global auth + global server + url = f'{server}api/users/{user_id}.json?fields=userGroups' + response = requests.get(url, auth=auth) + return response + + +def get_usergroups_filtered(user_id): + global auth + global server + url_usergroups = f'{server}api/userGroups.json?fields=id&filter=users.id:eq:{user_id}&paging=false' + response_usergroups = requests.get(url_usergroups, auth=auth) + return response_usergroups + + +def order(item): + if 'userGroups' in item: + item['userGroups'] = sorted(item['userGroups'], key=lambda x: x['id']) + return item + + +def order_json(value): + value = order(value) + return value + + +def main(config_path): + global auth + global server + global changes_detected + with open(config_path, 'r') as conf_file: + config_data = json.load(conf_file) + server = config_data.get('server') + mode = config_data.get("mode") + if mode == "ALL": + user_ids = requests.get(server+all_users_request, auth=auth) + else: + user_ids = config_data.get("user_ids") + username = config_data.get('user') + password = config_data.get('password') + auth = HTTPBasicAuth(username, password) + control_file = config_data.get("control_file") + log_file = config_data.get("log_file") + + for user_id in user_ids: + if check_user_changes(user_id, control_file, log_file): + changes_detected = True + + if changes_detected: + print("USERGROUP ERROR DETECTED") + else: + print("NO ERROR DETECTED") + + +def check_user_changes(user_id, control_users_file, log_file): + response = get_user_groups(user_id) + response_usergroups = get_usergroups_filtered(user_id) + + if response.status_code == 200 and response_usergroups.status_code == 200: + value = order_json(response.json()) + usergroupsvalue = order_json(response_usergroups.json()) + + if value != usergroupsvalue: + now = datetime.now() + formatted_date_time = now.strftime("%d_%m_%Y_%H_%M") + print(value) + print(usergroupsvalue) + print(formatted_date_time) + print(f"User {user_id} changed") + print("------------------------") + + with open(control_users_file, 'w') as f: + f.write(formatted_date_time) + + with open(log_file, 'a') as f: + f.write(formatted_date_time) + f.write(value) + f.write(usergroupsvalue) + f.write("-----") + return True + return False + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="DHIS2 USER-USERGROUP notification") + parser.add_argument('--config', required=True, help="Config file path") + args = parser.parse_args() + main(args.config) diff --git a/DHIS2/usergroups_error_notificator/example_config.json b/DHIS2/usergroups_error_notificator/example_config.json new file mode 100644 index 00000000..b9203e0e --- /dev/null +++ b/DHIS2/usergroups_error_notificator/example_config.json @@ -0,0 +1,9 @@ +{ + "control_file": "absolute_path.txt", + "log_file": "absolute_path.txt", + "server": "https://server/dhis2/", + "mode": "ALL", + "user_ids": ["uid1", "uid2"], + "user": "username", + "password": "password" +} From ba61aba904c91bfa4bf28e6d66155e4900254fdf Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 2 Oct 2024 14:03:41 +0200 Subject: [PATCH 2/8] Fix api call and add logs --- .../check_usergroups_error.py | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/DHIS2/usergroups_error_notificator/check_usergroups_error.py b/DHIS2/usergroups_error_notificator/check_usergroups_error.py index 45e862e1..85809a78 100644 --- a/DHIS2/usergroups_error_notificator/check_usergroups_error.py +++ b/DHIS2/usergroups_error_notificator/check_usergroups_error.py @@ -7,7 +7,7 @@ changes_detected = False server = "" auth = "" -all_users_request = "/api/users.json?fields=id&paging=false" +all_users_request = "api/users.json?fields=id&paging=false" def get_user_groups(user_id): @@ -37,28 +37,37 @@ def order_json(value): return value +def getTime(): + now = datetime.now() + formatted_date_time = now.strftime("%d_%m_%Y_%H_%M") + return formatted_date_time + + def main(config_path): global auth global server global changes_detected + + print("Starting at:" + getTime()) with open(config_path, 'r') as conf_file: config_data = json.load(conf_file) server = config_data.get('server') mode = config_data.get("mode") - if mode == "ALL": - user_ids = requests.get(server+all_users_request, auth=auth) - else: - user_ids = config_data.get("user_ids") username = config_data.get('user') password = config_data.get('password') auth = HTTPBasicAuth(username, password) + if mode == "ALL": + user_ids = requests.get(server+all_users_request, auth=auth).json() + else: + user_ids = config_data.get("user_ids") control_file = config_data.get("control_file") log_file = config_data.get("log_file") - for user_id in user_ids: - if check_user_changes(user_id, control_file, log_file): + for user_id in user_ids["users"]: + if check_user_changes(user_id["id"], control_file, log_file): changes_detected = True + print(getTime()) if changes_detected: print("USERGROUP ERROR DETECTED") else: @@ -74,8 +83,7 @@ def check_user_changes(user_id, control_users_file, log_file): usergroupsvalue = order_json(response_usergroups.json()) if value != usergroupsvalue: - now = datetime.now() - formatted_date_time = now.strftime("%d_%m_%Y_%H_%M") + formatted_date_time = getTime() print(value) print(usergroupsvalue) print(formatted_date_time) @@ -87,10 +95,14 @@ def check_user_changes(user_id, control_users_file, log_file): with open(log_file, 'a') as f: f.write(formatted_date_time) + f.write("User usergroups") f.write(value) + f.write("Usergroups filtered by user") f.write(usergroupsvalue) f.write("-----") return True + else: + print(user_id + " OK") return False From ebce433d9479ab3002b2e03c11051553fae23b7e Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 16 Oct 2024 12:51:27 +0200 Subject: [PATCH 3/8] remove older script due was more heavy way to do it for the servers --- .../check_usergroups_error.py | 114 ------------------ 1 file changed, 114 deletions(-) delete mode 100644 DHIS2/usergroups_error_notificator/check_usergroups_error.py diff --git a/DHIS2/usergroups_error_notificator/check_usergroups_error.py b/DHIS2/usergroups_error_notificator/check_usergroups_error.py deleted file mode 100644 index 85809a78..00000000 --- a/DHIS2/usergroups_error_notificator/check_usergroups_error.py +++ /dev/null @@ -1,114 +0,0 @@ -import json -from datetime import datetime -import requests -from requests.auth import HTTPBasicAuth -import argparse - -changes_detected = False -server = "" -auth = "" -all_users_request = "api/users.json?fields=id&paging=false" - - -def get_user_groups(user_id): - global auth - global server - url = f'{server}api/users/{user_id}.json?fields=userGroups' - response = requests.get(url, auth=auth) - return response - - -def get_usergroups_filtered(user_id): - global auth - global server - url_usergroups = f'{server}api/userGroups.json?fields=id&filter=users.id:eq:{user_id}&paging=false' - response_usergroups = requests.get(url_usergroups, auth=auth) - return response_usergroups - - -def order(item): - if 'userGroups' in item: - item['userGroups'] = sorted(item['userGroups'], key=lambda x: x['id']) - return item - - -def order_json(value): - value = order(value) - return value - - -def getTime(): - now = datetime.now() - formatted_date_time = now.strftime("%d_%m_%Y_%H_%M") - return formatted_date_time - - -def main(config_path): - global auth - global server - global changes_detected - - print("Starting at:" + getTime()) - with open(config_path, 'r') as conf_file: - config_data = json.load(conf_file) - server = config_data.get('server') - mode = config_data.get("mode") - username = config_data.get('user') - password = config_data.get('password') - auth = HTTPBasicAuth(username, password) - if mode == "ALL": - user_ids = requests.get(server+all_users_request, auth=auth).json() - else: - user_ids = config_data.get("user_ids") - control_file = config_data.get("control_file") - log_file = config_data.get("log_file") - - for user_id in user_ids["users"]: - if check_user_changes(user_id["id"], control_file, log_file): - changes_detected = True - - print(getTime()) - if changes_detected: - print("USERGROUP ERROR DETECTED") - else: - print("NO ERROR DETECTED") - - -def check_user_changes(user_id, control_users_file, log_file): - response = get_user_groups(user_id) - response_usergroups = get_usergroups_filtered(user_id) - - if response.status_code == 200 and response_usergroups.status_code == 200: - value = order_json(response.json()) - usergroupsvalue = order_json(response_usergroups.json()) - - if value != usergroupsvalue: - formatted_date_time = getTime() - print(value) - print(usergroupsvalue) - print(formatted_date_time) - print(f"User {user_id} changed") - print("------------------------") - - with open(control_users_file, 'w') as f: - f.write(formatted_date_time) - - with open(log_file, 'a') as f: - f.write(formatted_date_time) - f.write("User usergroups") - f.write(value) - f.write("Usergroups filtered by user") - f.write(usergroupsvalue) - f.write("-----") - return True - else: - print(user_id + " OK") - return False - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="DHIS2 USER-USERGROUP notification") - parser.add_argument('--config', required=True, help="Config file path") - args = parser.parse_args() - main(args.config) From 568415570a738a2933e99addabb09adfb383f737 Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 16 Oct 2024 12:52:08 +0200 Subject: [PATCH 4/8] added script to identify and fix the usergroup cache issue --- DHIS2/usergroups_error_notificator/Readme.md | 48 +++++ .../check_user_usergroup_size.py | 188 ++++++++++++++++++ .../example_config.json | 4 - 3 files changed, 236 insertions(+), 4 deletions(-) create mode 100644 DHIS2/usergroups_error_notificator/Readme.md create mode 100644 DHIS2/usergroups_error_notificator/check_user_usergroup_size.py diff --git a/DHIS2/usergroups_error_notificator/Readme.md b/DHIS2/usergroups_error_notificator/Readme.md new file mode 100644 index 00000000..a32cfa5a --- /dev/null +++ b/DHIS2/usergroups_error_notificator/Readme.md @@ -0,0 +1,48 @@ +# check_user_usergroup_size + +This script is designed to monitor changes in users and user groups on a DHIS2 server. Its primary goal is to detect cache issues and notify when they have been resolved. Additionally, it handles legitimate changes in user groups and updates the data accordingly. + +## How does the script work? + +### Configuration loading + +At the beginning, the script reads a configuration file that provides DHIS2 server credentials, including the server, username, and password. + +### Creating the initial comparison file + +If the `last_values.json` file does not exist, the script requests the current state of users and their groups from the server and saves it to this file. This ensures that there is always an initial state for future comparisons. + +### Avoiding false positives + +By filtering common users, the script ensures that any detected differences between the states are not due to new or removed users, but rather reflect actual changes in the user groups of existing users. This reduces the likelihood of false positives when detecting cache issues. + +### Detecting recent changes in user groups + +The script checks if there have been updates in the `userGroups` in the last 5 minutes. If recent changes are detected, the script ends as it doesn’t make sense to proceed if the user groups are being modified. + +### State comparison + +If no recent changes are found, the script compares the current state of users with the last saved state in `last_values.json`. If differences are detected, it could indicate a cache issue. + +### Cache cleaning + +If changes are detected, the script clears the DHIS2 server cache to see if this resolves the issue. After waiting for 30 seconds, it fetches the user data again. + +### Post-cache cleaning verification + +After cleaning the cache, the script compares the states again. If the issue is resolved (i.e., it was a cache issue), the new corrected state is saved, and the `control_file.json` is updated with the cache issue detection date. + +### Handling legitimate changes + +If the changes are not related to a cache issue, the script saves the new state without further modifications. + +## What files does the script manage? + +- **last_values.json**: Stores the last known state of users and their user groups for future comparisons. +- **control_file.json**: Stores the date when a cache issue was identified and resolved. This is used by Monit to trigger notifications. + +## How to run the script + +To execute the script, you need to pass the `--config` parameter followed by the path to the configuration file: + +`python check_user_usergroup_size.py --config file.json` diff --git a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py new file mode 100644 index 00000000..d614336d --- /dev/null +++ b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py @@ -0,0 +1,188 @@ +import json +import sys +import time +import requests +from requests.auth import HTTPBasicAuth +from datetime import datetime, timedelta +import argparse + +# Globals +server = "" +auth = "" +interval = 30 # Interval of 30 seconds between comparisons +last_values = "last_values.json" +usergroups_file = "usergroups.json" +control_file = "control_file.json" + + +def request(url): + response = requests.get(url, auth=auth) + return response + + +def clean_cache(): + return request(f'{server}api/38/maintenance?cacheClear=true') + + +def get_users_and_groups(): + """Fetches the list of users and their user group sizes.""" + return request(f'{server}api/users?fields=id,userGroups::size&paging=false') + + +def user_groups_updated_recently(): + """Checks if any user groups were updated in the last 5 minutes and in that case exit.""" + five_minutes_ago = (datetime.now() - timedelta(minutes=5) + ).strftime("%Y-%m-%dT%H:%M:%S") + response = request( + f'{server}api/userGroups?filter=lastUpdated:gt:{five_minutes_ago}&paging=false') + + if response.status_code == 200: + usergroups = response.json().get('userGroups', []) + return len(usergroups) > 0 + else: + print( + f"Error checking userGroups: {response.status_code} - {response.text}") + sys.exit(0) + + +def filter_common_users(current_users, previous_users): + """Filters out added or deleted users to compare only common ones to avoid false positives.""" + current_users_dict = {user['id']: user for user in current_users} + previous_users_dict = {user['id']: user for user in previous_users} + + common_user_ids = set(current_users_dict.keys()).intersection( + set(previous_users_dict.keys())) + filtered_current_users = [current_users_dict[user_id] + for user_id in common_user_ids] + filtered_previous_users = [previous_users_dict[user_id] + for user_id in common_user_ids] + + return filtered_current_users, filtered_previous_users + + +def save_state(filename, data): + """Saves the given data to the specified file.""" + with open(filename, 'w') as file: + json.dump(data, file, indent=4) + + +def load_state(filename): + """Loads and returns data from the specified file.""" + try: + with open(filename, 'r') as file: + return json.load(file) + except FileNotFoundError: + return {} + + +def states_are_different(state1, state2): + """Compares two states after filtering common users excluding new or deleted users.""" + state1, state2 = filter_common_users(state1, state2) + return state1 != state2 + + +def get_date(): + return datetime.now().strftime("%d_%m_%Y_%H_%M") + + +def handle_api_error(): + print("API request failed") + sys.exit(0) + + +def save_control_file(date): + """Saves the date in control file when a cache issue has been identified to be handled by monit.""" + control_data = {"last_cache_issue_date": date} + with open(control_file, 'w') as file: + json.dump(control_data, file, indent=4) + print(f"Saved cache issue date to {control_file}") + + +def compare_states_and_save(last_changed_data, current_data_after_wait, current_data_after_cache_clean): + """Handles the comparison of states and decides when to save.""" + if states_are_different(last_changed_data, current_data_after_cache_clean): + print( + "Clean cache fixed the problem. Saving the new state amd update control file.") + save_usergroups_to_analyze(get_date()) + save_state(last_values, current_data_after_cache_clean) + save_control_file(get_date()) + else: + print("No cache issue detected. Legitimate changes detected. Saving new state.") + save_state(last_values, current_data_after_cache_clean) + + +def save_usergroups_to_analyze(date): + """Saves the user groups for analysis.""" + response = request(f'{server}api/userGroups?fields=*&paging=false') + if response.status_code == 200: + usergroups_data = sorted(response.json().get( + 'userGroups', []), key=lambda x: x['id']) + save_state(date + usergroups_file, usergroups_data) + return response + + +def create_last_changed_file_if_not_exists(): + """Creates last_changed.json if it doesn't exist with the current server data.""" + if not load_state(last_values): # Check if file is empty or doesn't exist + print(f"{last_values} not found, creating it with the current state.") + response = get_users_and_groups() + if response.status_code == 200: + current_data = sorted( + response.json()["users"], key=lambda x: x['id']) + save_state(last_values, current_data) + print(f"Saved current state from the server to {last_values}.") + else: + print(f"Failed to fetch data from server to create {last_values}.") + sys.exit(0) + + +def main(config_path): + global auth + global server + + print("Starting at:" + get_date()) + + with open(config_path, 'r') as conf_file: + config_data = json.load(conf_file) + server = config_data.get('server') + auth = HTTPBasicAuth(config_data.get('user'), config_data.get('password')) + # This is used in the first execution. + create_last_changed_file_if_not_exists() + # Check if any user groups were updated recently and exit if true + if user_groups_updated_recently(): + print("User groups updated recently. Exiting.") + sys.exit(0) + + last_changed_data = load_state(last_values) + response = get_users_and_groups() + + if response.status_code != 200: + handle_api_error() + + current_data = sorted(response.json()["users"], key=lambda x: x['id']) + print("Comparing current state.") + + if states_are_different(last_changed_data, current_data): + print("Changes detected. Cleaning cache and waiting.") + clean_cache() + time.sleep(interval) + + new_response = get_users_and_groups() + if new_response.status_code != 200: + handle_api_error() + + current_data_after_cache_clean = sorted( + new_response.json()["users"], key=lambda x: x['id']) + + compare_states_and_save( + last_changed_data, current_data, current_data_after_cache_clean) + else: + print("No changes detected.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="DHIS2 User-Usergroup notification script") + parser.add_argument('--config', required=True, help="Config file path") + args = parser.parse_args() + main(args.config) diff --git a/DHIS2/usergroups_error_notificator/example_config.json b/DHIS2/usergroups_error_notificator/example_config.json index b9203e0e..cbe8aae7 100644 --- a/DHIS2/usergroups_error_notificator/example_config.json +++ b/DHIS2/usergroups_error_notificator/example_config.json @@ -1,9 +1,5 @@ { - "control_file": "absolute_path.txt", - "log_file": "absolute_path.txt", "server": "https://server/dhis2/", - "mode": "ALL", - "user_ids": ["uid1", "uid2"], "user": "username", "password": "password" } From 088b479e8f9341b55c4108c2bd6ec632fec7436b Mon Sep 17 00:00:00 2001 From: idelcano Date: Wed, 16 Oct 2024 13:09:11 +0200 Subject: [PATCH 5/8] fix last check --- .../check_user_usergroup_size.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py index d614336d..ac0257fe 100644 --- a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py +++ b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py @@ -98,17 +98,17 @@ def save_control_file(date): print(f"Saved cache issue date to {control_file}") -def compare_states_and_save(last_changed_data, current_data_after_wait, current_data_after_cache_clean): +def compare_states_and_save(data_before_clean_cache, data_after_cache_clean): """Handles the comparison of states and decides when to save.""" - if states_are_different(last_changed_data, current_data_after_cache_clean): + if states_are_different(data_before_clean_cache, data_after_cache_clean): print( "Clean cache fixed the problem. Saving the new state amd update control file.") save_usergroups_to_analyze(get_date()) - save_state(last_values, current_data_after_cache_clean) + save_state(last_values, data_after_cache_clean) save_control_file(get_date()) else: print("No cache issue detected. Legitimate changes detected. Saving new state.") - save_state(last_values, current_data_after_cache_clean) + save_state(last_values, data_after_cache_clean) def save_usergroups_to_analyze(date): @@ -174,8 +174,7 @@ def main(config_path): current_data_after_cache_clean = sorted( new_response.json()["users"], key=lambda x: x['id']) - compare_states_and_save( - last_changed_data, current_data, current_data_after_cache_clean) + compare_states_and_save(current_data, current_data_after_cache_clean) else: print("No changes detected.") From 5c29d66ccda22acd2cf8cf1745bbe44e9c64db49 Mon Sep 17 00:00:00 2001 From: idelcano Date: Thu, 24 Oct 2024 08:16:01 +0200 Subject: [PATCH 6/8] Added folder field to select where save all the files required to check the usergroup error --- DHIS2/usergroups_error_notificator/example_config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DHIS2/usergroups_error_notificator/example_config.json b/DHIS2/usergroups_error_notificator/example_config.json index cbe8aae7..994b5b41 100644 --- a/DHIS2/usergroups_error_notificator/example_config.json +++ b/DHIS2/usergroups_error_notificator/example_config.json @@ -1,5 +1,6 @@ { "server": "https://server/dhis2/", "user": "username", - "password": "password" + "password": "password", + "folder": "/path/to/save/control/files" } From d2fdd476d7c23b93a0b26247efb207dc855b2c02 Mon Sep 17 00:00:00 2001 From: idelcano Date: Thu, 24 Oct 2024 08:16:34 +0200 Subject: [PATCH 7/8] refactor script to avoid use global or a lot of parameters in each method using a class --- .../check_user_usergroup_size.py | 257 +++++++++--------- 1 file changed, 122 insertions(+), 135 deletions(-) diff --git a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py index ac0257fe..1c79d356 100644 --- a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py +++ b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py @@ -5,176 +5,163 @@ from requests.auth import HTTPBasicAuth from datetime import datetime, timedelta import argparse +import os -# Globals -server = "" -auth = "" -interval = 30 # Interval of 30 seconds between comparisons -last_values = "last_values.json" -usergroups_file = "usergroups.json" -control_file = "control_file.json" +class DHIS2Monitor: + def __init__(self, config): + self.server = config.get('server') + self.auth = HTTPBasicAuth(config.get('user'), config.get('password')) + self.interval = 30 # Interval of 30 seconds between comparisons + folder = config.get('folder') + self.last_values = os.path.join(folder, "last_values.json") + self.usergroups_file = os.path.join(folder, "usergroups.json") + self.control_file = os.path.join(folder, "control_file.json") -def request(url): - response = requests.get(url, auth=auth) - return response + def request(self, url): + response = requests.get(url, auth=self.auth) + return response + def clean_cache(self): + return self.request(f'{self.server}api/38/maintenance?cacheClear=true') -def clean_cache(): - return request(f'{server}api/38/maintenance?cacheClear=true') + def get_users_and_groups(self): + """Fetches the list of users and their user group sizes.""" + return self.request(f'{self.server}api/users?fields=id,userGroups::size&paging=false') + def user_groups_updated_recently(self): + """Checks if any user groups were updated in the last 5 minutes and in that case exit.""" + five_minutes_ago = (datetime.now() - timedelta(minutes=5) + ).strftime("%Y-%m-%dT%H:%M:%S") + response = self.request( + f'{self.server}api/userGroups?filter=lastUpdated:gt:{five_minutes_ago}&paging=false') -def get_users_and_groups(): - """Fetches the list of users and their user group sizes.""" - return request(f'{server}api/users?fields=id,userGroups::size&paging=false') - - -def user_groups_updated_recently(): - """Checks if any user groups were updated in the last 5 minutes and in that case exit.""" - five_minutes_ago = (datetime.now() - timedelta(minutes=5) - ).strftime("%Y-%m-%dT%H:%M:%S") - response = request( - f'{server}api/userGroups?filter=lastUpdated:gt:{five_minutes_ago}&paging=false') + if response.status_code == 200: + usergroups = response.json().get('userGroups', []) + return len(usergroups) > 0 + else: + print(f"Error checking userGroups: {response.status_code} - {response.text}") + sys.exit(0) - if response.status_code == 200: - usergroups = response.json().get('userGroups', []) - return len(usergroups) > 0 - else: - print( - f"Error checking userGroups: {response.status_code} - {response.text}") + def filter_common_users(self, current_users, previous_users): + """Filters out added or deleted users to compare only common ones to avoid false positives.""" + current_users_dict = {user['id']: user for user in current_users} + previous_users_dict = {user['id']: user for user in previous_users} + + common_user_ids = set(current_users_dict.keys()).intersection( + set(previous_users_dict.keys())) + filtered_current_users = [current_users_dict[user_id] + for user_id in common_user_ids] + filtered_previous_users = [previous_users_dict[user_id] + for user_id in common_user_ids] + + return filtered_current_users, filtered_previous_users + + def save_state(self, filename, data): + """Saves the given data to the specified file.""" + with open(filename, 'w') as file: + json.dump(data, file, indent=4) + + def load_state(self, filename): + """Loads and returns data from the specified file.""" + try: + with open(filename, 'r') as file: + return json.load(file) + except FileNotFoundError: + return {} + + def states_are_different(self, state1, state2): + """Compares two states after filtering common users excluding new or deleted users.""" + state1, state2 = self.filter_common_users(state1, state2) + return state1 != state2 + + def get_date(self): + return datetime.now().strftime("%d_%m_%Y_%H_%M") + + def handle_api_error(self): + print("API request failed") sys.exit(0) + def save_control_file(self, date): + """Saves the date in control file when a cache issue has been identified to be handled by monit.""" + control_data = {"last_cache_issue_date": date} + with open(self.control_file, 'w') as file: + json.dump(control_data, file, indent=4) + print(f"Saved cache issue date to {self.control_file}") + + def compare_states_and_save(self, data_before_clean_cache, data_after_cache_clean): + """Handles the comparison of states and decides when to save.""" + if self.states_are_different(data_before_clean_cache, data_after_cache_clean): + print("Clean cache fixed the problem. Saving the new state and updating control file.") + self.save_usergroups_to_analyze(self.get_date()) + self.save_state(self.last_values, data_after_cache_clean) + self.save_control_file(self.get_date()) + else: + print("No cache issue detected. Legitimate changes detected. Saving new state.") + self.save_state(self.last_values, data_after_cache_clean) -def filter_common_users(current_users, previous_users): - """Filters out added or deleted users to compare only common ones to avoid false positives.""" - current_users_dict = {user['id']: user for user in current_users} - previous_users_dict = {user['id']: user for user in previous_users} - - common_user_ids = set(current_users_dict.keys()).intersection( - set(previous_users_dict.keys())) - filtered_current_users = [current_users_dict[user_id] - for user_id in common_user_ids] - filtered_previous_users = [previous_users_dict[user_id] - for user_id in common_user_ids] - - return filtered_current_users, filtered_previous_users - - -def save_state(filename, data): - """Saves the given data to the specified file.""" - with open(filename, 'w') as file: - json.dump(data, file, indent=4) - - -def load_state(filename): - """Loads and returns data from the specified file.""" - try: - with open(filename, 'r') as file: - return json.load(file) - except FileNotFoundError: - return {} - - -def states_are_different(state1, state2): - """Compares two states after filtering common users excluding new or deleted users.""" - state1, state2 = filter_common_users(state1, state2) - return state1 != state2 - - -def get_date(): - return datetime.now().strftime("%d_%m_%Y_%H_%M") - - -def handle_api_error(): - print("API request failed") - sys.exit(0) - - -def save_control_file(date): - """Saves the date in control file when a cache issue has been identified to be handled by monit.""" - control_data = {"last_cache_issue_date": date} - with open(control_file, 'w') as file: - json.dump(control_data, file, indent=4) - print(f"Saved cache issue date to {control_file}") - - -def compare_states_and_save(data_before_clean_cache, data_after_cache_clean): - """Handles the comparison of states and decides when to save.""" - if states_are_different(data_before_clean_cache, data_after_cache_clean): - print( - "Clean cache fixed the problem. Saving the new state amd update control file.") - save_usergroups_to_analyze(get_date()) - save_state(last_values, data_after_cache_clean) - save_control_file(get_date()) - else: - print("No cache issue detected. Legitimate changes detected. Saving new state.") - save_state(last_values, data_after_cache_clean) - - -def save_usergroups_to_analyze(date): - """Saves the user groups for analysis.""" - response = request(f'{server}api/userGroups?fields=*&paging=false') - if response.status_code == 200: - usergroups_data = sorted(response.json().get( - 'userGroups', []), key=lambda x: x['id']) - save_state(date + usergroups_file, usergroups_data) - return response - - -def create_last_changed_file_if_not_exists(): - """Creates last_changed.json if it doesn't exist with the current server data.""" - if not load_state(last_values): # Check if file is empty or doesn't exist - print(f"{last_values} not found, creating it with the current state.") - response = get_users_and_groups() + def save_usergroups_to_analyze(self, date): + """Saves the user groups for analysis.""" + response = self.request(f'{self.server}api/userGroups?fields=*&paging=false') if response.status_code == 200: - current_data = sorted( - response.json()["users"], key=lambda x: x['id']) - save_state(last_values, current_data) - print(f"Saved current state from the server to {last_values}.") - else: - print(f"Failed to fetch data from server to create {last_values}.") - sys.exit(0) + usergroups_data = sorted(response.json().get( + 'userGroups', []), key=lambda x: x['id']) + self.save_state(f"{date}_{self.usergroups_file}", usergroups_data) + return response + + def create_last_changed_file_if_not_exists(self): + """Creates last_changed.json if it doesn't exist with the current server data.""" + if not self.load_state(self.last_values): # Check if file is empty or doesn't exist + print(f"{self.last_values} not found, creating it with the current state.") + response = self.get_users_and_groups() + if response.status_code == 200: + current_data = sorted( + response.json()["users"], key=lambda x: x['id']) + self.save_state(self.last_values, current_data) + print(f"Saved current state from the server to {self.last_values}.") + else: + print(f"Failed to fetch data from server to create {self.last_values}.") + sys.exit(0) def main(config_path): - global auth - global server - - print("Starting at:" + get_date()) + print("Starting at:" + datetime.now().strftime("%d_%m_%Y_%H_%M")) with open(config_path, 'r') as conf_file: config_data = json.load(conf_file) - server = config_data.get('server') - auth = HTTPBasicAuth(config_data.get('user'), config_data.get('password')) + + monitor = DHIS2Monitor(config_data) + # This is used in the first execution. - create_last_changed_file_if_not_exists() + monitor.create_last_changed_file_if_not_exists() + # Check if any user groups were updated recently and exit if true - if user_groups_updated_recently(): + if monitor.user_groups_updated_recently(): print("User groups updated recently. Exiting.") sys.exit(0) - last_changed_data = load_state(last_values) - response = get_users_and_groups() + last_changed_data = monitor.load_state(monitor.last_values) + response = monitor.get_users_and_groups() if response.status_code != 200: - handle_api_error() + monitor.handle_api_error() current_data = sorted(response.json()["users"], key=lambda x: x['id']) print("Comparing current state.") - if states_are_different(last_changed_data, current_data): + if monitor.states_are_different(last_changed_data, current_data): print("Changes detected. Cleaning cache and waiting.") - clean_cache() - time.sleep(interval) + monitor.clean_cache() + time.sleep(monitor.interval) - new_response = get_users_and_groups() + new_response = monitor.get_users_and_groups() if new_response.status_code != 200: - handle_api_error() + monitor.handle_api_error() current_data_after_cache_clean = sorted( new_response.json()["users"], key=lambda x: x['id']) - compare_states_and_save(current_data, current_data_after_cache_clean) + monitor.compare_states_and_save(current_data, current_data_after_cache_clean) else: print("No changes detected.") @@ -184,4 +171,4 @@ def main(config_path): description="DHIS2 User-Usergroup notification script") parser.add_argument('--config', required=True, help="Config file path") args = parser.parse_args() - main(args.config) + main(args.config) \ No newline at end of file From 2708f1d6fb9bac31ce54cf648a8d161965ecfc1f Mon Sep 17 00:00:00 2001 From: idelcano Date: Thu, 24 Oct 2024 08:18:02 +0200 Subject: [PATCH 8/8] update readme --- DHIS2/usergroups_error_notificator/check_user_usergroup_size.py | 1 - 1 file changed, 1 deletion(-) diff --git a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py index 1c79d356..dff7cb25 100644 --- a/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py +++ b/DHIS2/usergroups_error_notificator/check_user_usergroup_size.py @@ -132,7 +132,6 @@ def main(config_path): monitor = DHIS2Monitor(config_data) - # This is used in the first execution. monitor.create_last_changed_file_if_not_exists() # Check if any user groups were updated recently and exit if true