From 19dbb8a0ec5ac579c8820f5815b0ba11ed5a3f6f Mon Sep 17 00:00:00 2001 From: idelcano Date: Mon, 28 Dec 2020 18:19:54 +0100 Subject: [PATCH 1/3] Added config and git ignore files --- .../orgunit_programs_appender/config.json | 5 +++++ .../orgunit_programs_appender/input/.gitignore | 1 + .../orgunit_programs_appender/output/.gitignore | 1 + 3 files changed, 7 insertions(+) create mode 100644 DHIS2/metadata_manipulation/orgunit_programs_appender/config.json create mode 100644 DHIS2/metadata_manipulation/orgunit_programs_appender/input/.gitignore create mode 100644 DHIS2/metadata_manipulation/orgunit_programs_appender/output/.gitignore diff --git a/DHIS2/metadata_manipulation/orgunit_programs_appender/config.json b/DHIS2/metadata_manipulation/orgunit_programs_appender/config.json new file mode 100644 index 00000000..caede551 --- /dev/null +++ b/DHIS2/metadata_manipulation/orgunit_programs_appender/config.json @@ -0,0 +1,5 @@ +{ + "user": "", + "password": "", + "server": "" +} \ No newline at end of file diff --git a/DHIS2/metadata_manipulation/orgunit_programs_appender/input/.gitignore b/DHIS2/metadata_manipulation/orgunit_programs_appender/input/.gitignore new file mode 100644 index 00000000..16f2dc5f --- /dev/null +++ b/DHIS2/metadata_manipulation/orgunit_programs_appender/input/.gitignore @@ -0,0 +1 @@ +*.csv \ No newline at end of file diff --git a/DHIS2/metadata_manipulation/orgunit_programs_appender/output/.gitignore b/DHIS2/metadata_manipulation/orgunit_programs_appender/output/.gitignore new file mode 100644 index 00000000..94a2dd14 --- /dev/null +++ b/DHIS2/metadata_manipulation/orgunit_programs_appender/output/.gitignore @@ -0,0 +1 @@ +*.json \ No newline at end of file From bd848f8e1bc47cd45f1654bf59677f702dce1cc4 Mon Sep 17 00:00:00 2001 From: idelcano Date: Mon, 28 Dec 2020 18:20:16 +0100 Subject: [PATCH 2/3] Added orgunit_program_appender --- .../orgunit_program_appender.py | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py diff --git a/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py b/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py new file mode 100644 index 00000000..83af4f12 --- /dev/null +++ b/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py @@ -0,0 +1,201 @@ +import csv +import json +import os +import sys +from datetime import time +from enum import Enum +from os import listdir +from os.path import isfile, join + +from DHIS2.cloner import dhis2api + +input_dir = "input" +output_dir = "output" +config_file = "config.json" +query_programs = "/metadata.json?filter=id:in:[%s]&paging=false" +query_orgunit_levels = "/organisationUnits.json?filter=path:like:%s/&filter=level:eq:%d&fields=id&paging=false" +query_orgunit = "/organisationUnits/%s.json?fields=id&paging=false" +query_orgunit_children = "/organisationUnits/%s.json?fields=children&paging=false" +query_orgunit_descendants = "/organisationUnits/%s.json?includeDescendants=true&fields=id&paging=false" +query_orgunitgroups = "/organisationUnitGroups/%s.json?fields=id&paging=false" + + +class Children(Enum): + no = 1 + yes = 2 + level = 3 + alldescendants = 4 + orgunitgroup = 5 + + +def assign_fields(ou, dataset, eventprogram, trackerprogram, children, replace, data): + if ou is None: + print("Ou field is required") + sys.exit() + if children is None: + print("With children field is required") + sys.exit() + + if not dataset and not eventprogram and not trackerprogram : + print("At least one dataset, event program, or tracker program is required") + sys.exit() + if dataset: + data["dataSets"].append(dataset) + identifier = dataset + if eventprogram: + data["programs"].append(eventprogram) + identifier = eventprogram + if trackerprogram: + data["programs"].append(trackerprogram) + identifier = trackerprogram + + if replace is None or replace.lower() == "no": + replace = False + elif replace.lower() == "yes": + replace = True + else: + print("replace field is not valid (Yes / No )") + sys.exit() + data["items"].append({"identifier": identifier, "children": children, "orgUnit": ou, "replace": replace, "query": ""}) + + +def get_data_from_csv(files): + data = {"programs": [], "dataSets": [], "items": []} + for path_file in files: + with open(path_file) as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + line_count = 0 + for row in csv_reader: + if line_count == 0: + line_count += 1 + else: + line_count += 1 + ou = row[0] + children = row[1].lower() + dataset = row[2] + eventprogram = row[3] + trackerprogram = row[4] + replace = row[5] + assign_fields(ou, dataset, eventprogram, trackerprogram, children, replace, data) + return data + + +def generate_queries(data): + for item in data["items"]: + children = item["children"] + org_unit_uid = item["orgUnit"] + if Children.no.name == children: + query = query_orgunit % (item["orgUnit"]) + elif Children.level.name in children: + import re + level = re.findall(r'\d+', children) + query = query_orgunit_children % (org_unit_uid, level) + elif Children.yes.name == children: + query = query_orgunit_children % org_unit_uid + elif Children.alldescendants.name == children: + query = query_orgunit_descendants % org_unit_uid + elif Children.orgunitgroup.name == children: + query = query_orgunitgroups % org_unit_uid + item["query"] = query + + +def append_org_units(programs_from_server, data): + for item in data: + append_in_metadata(item, programs_from_server, "programs") + append_in_metadata(item, programs_from_server, "dataSets") + + +def append_in_metadata(item, programs_from_server, key): + for metadata_item in programs_from_server[key]: + if item["identifier"] == metadata_item["id"]: + if item["replace"]: + metadata_item["organisationUnits"] = item["organisationUnits"] + else: + item_orgunits = set(item["organisationUnits"]) + program_orgunits = set(metadata_item["organisationUnits"]) + new_orgunits = list(item_orgunits - program_orgunits) + metadata_item["organisationUnits"] = program_orgunits + new_orgunits + + +def execute_queries(api, data): + for item in data: + children = item["children"] + query = item["query"] + orgunits = api.get(query) + #sleep some seconds between api calls + time.sleep(3) + + if Children.no == children: + item["organisationUnits"] = {"id": orgunits["id"]} + elif Children.yes.name == children: + item["organisationUnits"] = orgunits["children"] + elif Children.level.name == children or Children.alldescendants.name == children: + item["organisationUnits"] = orgunits["organisationUnits"] + elif Children.orgunitgroup.name == children: + for orgunitgroup in orgunits["organisationUnitGroup"]: + item["organisationUnits"] = orgunitgroup["organisationUnits"] + + + +def main(): + global paths + global uid_list + global input_data + input_data = dict() + path = os.path.abspath(__file__) + applied_filter = lambda fname: fname.endswith('.csv') + if "/" in path: + path = path[:path.rfind('/')+1] + elif "\\" in path: + path = path[:path.rfind('\\') + 1] + fixed_input_dir = join(path, input_dir) + fixed_output_dir = join(path, output_dir) + files = [join(fixed_input_dir, f) for f in filter(applied_filter, listdir(fixed_input_dir)) if isfile(join(fixed_input_dir, f))] + cfg = get_config(join(path,config_file)) + user = cfg["user"] + password = cfg["password"] + server = cfg["server"] + + if not user or not password or not server: + print("ERROR: Please provide server, username and password") + sys.exit() + input_data = get_data_from_csv(files) + + api = dhis2api.Dhis2Api(server, user, password) + + programs_from_server = get_programs_from_server(api, input_data) + generate_queries(input_data) + execute_queries(api, input_data) + append_org_units(programs_from_server, input_data) + write_to_json(fixed_output_dir, programs_from_server) + + +def get_programs_from_server(api, data): + program_uids = "" + for dataset in data["dataSets"]: + program_uids = program_uids + dataset + "," + for program in data["programs"]: + program_uids = program_uids + program + "," + program_uids = program_uids[:program_uids.rfind(',')] + return api.get(query_programs % (program_uids)) + + +def write_to_json(file, content): + from datetime import datetime + with open(join(file + "importer_"+ datetime.now() + ".json"), 'w', encoding='utf-8') as file: + json.dump(content, file, indent=4) + + +def get_config(fname): + "Return dict with the options read from configuration file" + print('Reading from config file %s ...' % fname) + try: + with open(fname) as f: + config = json.load(f) + except (AssertionError, IOError, ValueError) as e: + sys.exit('Error reading config file %s: %s' % (fname, e)) + return config + + +if __name__ == '__main__': + main() From fa513ef36f1077f44c418c0d283ae247042f716d Mon Sep 17 00:00:00 2001 From: idelcano Date: Mon, 28 Dec 2020 19:14:53 +0100 Subject: [PATCH 3/3] Fix some bugs --- .../orgunit_program_appender.py | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py b/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py index 83af4f12..088d9bb8 100644 --- a/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py +++ b/DHIS2/metadata_manipulation/orgunit_programs_appender/orgunit_program_appender.py @@ -2,7 +2,8 @@ import json import os import sys -from datetime import time +import time as time +from datetime import datetime as date from enum import Enum from os import listdir from os.path import isfile, join @@ -17,7 +18,7 @@ query_orgunit = "/organisationUnits/%s.json?fields=id&paging=false" query_orgunit_children = "/organisationUnits/%s.json?fields=children&paging=false" query_orgunit_descendants = "/organisationUnits/%s.json?includeDescendants=true&fields=id&paging=false" -query_orgunitgroups = "/organisationUnitGroups/%s.json?fields=id&paging=false" +query_orgunitgroups = "/organisationUnitGroups/%s.json?fields=organisationUnits&paging=false" class Children(Enum): @@ -88,8 +89,8 @@ def generate_queries(data): query = query_orgunit % (item["orgUnit"]) elif Children.level.name in children: import re - level = re.findall(r'\d+', children) - query = query_orgunit_children % (org_unit_uid, level) + level = int(re.findall(r'\d+', children)[0]) + query = query_orgunit_levels % (org_unit_uid, level) elif Children.yes.name == children: query = query_orgunit_children % org_unit_uid elif Children.alldescendants.name == children: @@ -100,7 +101,7 @@ def generate_queries(data): def append_org_units(programs_from_server, data): - for item in data: + for item in data["items"]: append_in_metadata(item, programs_from_server, "programs") append_in_metadata(item, programs_from_server, "dataSets") @@ -111,30 +112,25 @@ def append_in_metadata(item, programs_from_server, key): if item["replace"]: metadata_item["organisationUnits"] = item["organisationUnits"] else: - item_orgunits = set(item["organisationUnits"]) - program_orgunits = set(metadata_item["organisationUnits"]) - new_orgunits = list(item_orgunits - program_orgunits) - metadata_item["organisationUnits"] = program_orgunits + new_orgunits + for item_org_unit in item["organisationUnits"]: + if item_org_unit not in metadata_item["organisationUnits"]: + metadata_item["organisationUnits"].append(item_org_unit) def execute_queries(api, data): - for item in data: + for item in data["items"]: children = item["children"] query = item["query"] orgunits = api.get(query) #sleep some seconds between api calls time.sleep(3) - if Children.no == children: - item["organisationUnits"] = {"id": orgunits["id"]} + if Children.no.name == children: + item["organisationUnits"] = [{"id": orgunits["id"]}] elif Children.yes.name == children: item["organisationUnits"] = orgunits["children"] - elif Children.level.name == children or Children.alldescendants.name == children: + else: item["organisationUnits"] = orgunits["organisationUnits"] - elif Children.orgunitgroup.name == children: - for orgunitgroup in orgunits["organisationUnitGroup"]: - item["organisationUnits"] = orgunitgroup["organisationUnits"] - def main(): @@ -180,9 +176,9 @@ def get_programs_from_server(api, data): return api.get(query_programs % (program_uids)) -def write_to_json(file, content): - from datetime import datetime - with open(join(file + "importer_"+ datetime.now() + ".json"), 'w', encoding='utf-8') as file: +def write_to_json(path, content): + file_name = "result_" + str(date.now()).replace(" ", "").replace(":", "-").split(".")[0] + ".json" + with open(join(join(path, file_name)), 'w', encoding='utf-8') as file: json.dump(content, file, indent=4)