From 0ebb1704b457d673ba460b0496305f44d23380d0 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 08:58:59 +0100 Subject: [PATCH 01/40] #4036: Clone tool/core into cycle dir. Symlink into python/lib --- .../app/get_esmval/opt/rose-app-metoffice.conf | 8 ++++++++ .../recipe_test_workflow/site/metoffice/runtime.cylc | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf index 8911ca1cfd..4a6a5c3472 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf @@ -5,3 +5,11 @@ default=clone_latest_esmval.sh BRANCH=main ESMVALCORE_URL=git@github.com:ESMValGroup/ESMValCore.git ESMVALTOOL_URL=git@github.com:ESMValGroup/ESMValTool.git + +[file:${CYLC_WORKFLOW_SHARE_DIR}/lib/python/ESMValCore] +source=${ROSE_DATAC}/ESMValCore/ +mode=symlink + +[file:${CYLC_WORKFLOW_SHARE_DIR}/lib/python/ESMValTool] +source=${ROSE_DATAC}/ESMValTool/ +mode=symlink diff --git a/esmvaltool/utils/recipe_test_workflow/site/metoffice/runtime.cylc b/esmvaltool/utils/recipe_test_workflow/site/metoffice/runtime.cylc index c1589093d6..7ae97b7528 100644 --- a/esmvaltool/utils/recipe_test_workflow/site/metoffice/runtime.cylc +++ b/esmvaltool/utils/recipe_test_workflow/site/metoffice/runtime.cylc @@ -2,8 +2,8 @@ [runtime] [[root]] [[[environment]]] - ESMVALCORE_DIR = ${CYLC_WORKFLOW_RUN_DIR}/share/lib/python/ESMValCore - ESMVALTOOL_DIR = ${CYLC_WORKFLOW_RUN_DIR}/share/lib/python/ESMValTool + ESMVALCORE_DIR = ${ROSE_DATAC}/ESMValCore + ESMVALTOOL_DIR = ${ROSE_DATAC}/ESMValTool PYTHONPATH_PREPEND = ${ESMVALCORE_DIR}:${ESMVALTOOL_DIR} # COMPUTE provides defaults for computation-heavy tasks. From 2fc7ecc21663ac51b45de56c1ddf3811594868af Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 09:01:26 +0100 Subject: [PATCH 02/40] #4036: Configure rose to set previous cycle point env var --- esmvaltool/utils/recipe_test_workflow/flow.cylc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index fcfb8df6bc..8a41d9c06b 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -51,7 +51,7 @@ [runtime] [[root]] script = rose task-run - env-script = "eval $(rose task-env)" + env-script = "eval $(rose task-env --cycle-offset=P1D)" [[[environment]]] ENV_NAME = {{ ENV_NAME }} USER_CONFIG_DIR = ${ROSE_DATAC}/config_dir From 9ebd0f2364e707384acc862d9187381afcbbf2e6 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 09:08:56 +0100 Subject: [PATCH 03/40] #4036: Add git logs to report. Includes dev run local option. --- .../bin/generate_html_report.py | 148 +++++++++++++++++- .../generate_report/bin/report_template.jinja | 61 +++++++- 2 files changed, 205 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index ded65bd0ba..fdda333232 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -2,14 +2,52 @@ import os import sqlite3 from datetime import datetime +from pathlib import Path +import subprocess from jinja2 import Environment, FileSystemLoader, select_autoescape +# UNCOMMENT FOR LOCAL +# CYLC_DB_PATH = "hello" +# CYLC_TASK_CYCLE_POINT = "20250516T1053Z" +# REPORT_PATH="/home/users/christopher.billows/Code/ESMValTool/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report.html" +# cached_raw_db_data = [ +# ('get_esmval', 'waiting'), +# ('install_env_file','succeeded'), +# ('get_esmval','succeeded'), +# ('configure','succeeded'), +# ('compare_recipe_radiation_budget','succeeded'), +# ('process_recipe_radiation_budget','succeeded'), +# ('process_recipe_albedolandcover','succeeded'), +# ('process_recipe_ocean_amoc','succeeded'), +# ('process_recipe_autoassess_landsurface_soilmoisture','succeeded'), +# ('process_recipe_heatwaves_coldwaves','succeeded'), +# ('process_recipe_ocean_multimap','succeeded'), +# ('process_recipe_ensclus','succeeded'), +# ('process_recipe_consecdrydays','succeeded'), +# ('compare_recipe_albedolandcover','succeeded'), +# ('compare_recipe_consecdrydays','succeeded'), +# ('generate_report','running'), +# ('compare_recipe_autoassess_landsurface_soilmoisture','succeeded'), +# ('compare_recipe_heatwaves_coldwaves','succeeded'), +# ('compare_recipe_ensclus','succeeded'), +# ('compare_recipe_ocean_multimap','succeeded'), +# ('compare_recipe_ocean_amoc','succeeded') +# ] + + +# UNCOMMENT FOR REAL CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") -CYLC_WORKFLOW_SHARE_DIR = os.environ.get("CYLC_WORKFLOW_SHARE_DIR") +CYLC_TASK_PREVIOUS_CYCLE = os.environ.get("ROSE_DATACP1D") REPORT_PATH = os.environ.get("REPORT_PATH") +ESMVAL_CORE_CURRENT = os.environ.get("ESMVALCORE_DIR") +ESMVAL_TOOL_CURRENT = os.environ.get("ESMVALTOOL_DIR") + +ESMVAL_CORE_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValCore" +ESMVAL_TOOL_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValTool" + SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" @@ -23,12 +61,50 @@ def main(db_file_path=CYLC_DB_PATH): db_file_path : str, default CYLC_DB_FILE_PATH The path to the SQLite database file. """ + + # UNCOMMENT FOR LOCAL + # raw_db_data = cached_raw_db_data + # processed_db_data = process_db_output(raw_db_data) + # local_esmvaltool = Path("/home/users/christopher.billows/Code/ESMValTool/") + # local_esmvalcore = Path("/home/users/christopher.billows/Code/ESMValCore/") + # esmval_core_previous_commit_sha = "170a93893" + # esmval_tool_previous_commit_sha = "4515a2b92" + # esmval_core_all_commits = fetch_git_commits( + # local_esmvalcore, esmval_core_previous_commit_sha + # ) + # esmval_tool_all_commits = fetch_git_commits( + # local_esmvaltool, esmval_tool_previous_commit_sha + # ) + + # UNCOMMENT FOR REAL raw_db_data = fetch_report_data(db_file_path) processed_db_data = process_db_output(raw_db_data) + + esmval_core_previous_commit_sha = None + if ESMVAL_CORE_PREVIOUS.exists(): + esmval_core_previous_commit_sha = ( + fetch_git_commits(ESMVAL_CORE_PREVIOUS)['sha'] + ) + + esmval_tool_previous_commit_sha = None + if ESMVAL_TOOL_PREVIOUS.exists(): + esmval_tool_previous_commit_sha = ( + fetch_git_commits(ESMVAL_TOOL_PREVIOUS)['sha'] + ) + + esmval_core_all_commits = fetch_git_commits( + ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha + ) + esmval_tool_all_commits = fetch_git_commits( + ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha + ) + subheader = create_subheader() rendered_html = render_html_report( subheader=subheader, report_data=processed_db_data, + esmval_core_commits=esmval_core_all_commits, + esmval_tool_commits=esmval_tool_all_commits, ) write_report_to_file(rendered_html) @@ -143,6 +219,66 @@ def process_db_output(report_data): return sorted_processed_db_data +def add_report_message_to_git_commits(git_commits_info): + """ + Add report messages to a git commit information dictionary. + + Parameters + ---------- + list[dict] + A list of git commits. + """ + git_commits_info[0]['report_flag'] = "Version tested >>>" + if len(git_commits_info) > 1: + git_commits_info[-1]['report_flag'] = "Version last tested >>>" + + +def fetch_git_commits(package_path, sha=None): + """ + Fetch git commit information for an installed package. + + Parameters + ---------- + package_path : str + Path to a package's git repo. + sha: str | None + Optional. The sha of a previously tested commit. If provided, commits + from HEAD back to the passed sha (inclusive) will be retrieved. + + Returns + ------- + list[dict] + A list of dicts where each dict represents one commit. If ``sha`` is + passed, multiple commits/dicts may be returned. + """ + command = [ + "git", "log", "-1", "--date=iso-strict", "--pretty=%cd^_^%h^_^%an^_^%s" + ] + + if sha: + command[2] = f"{sha}^..HEAD" + + raw_commit_info = subprocess.run( + command, cwd=package_path, capture_output=True, check=True, text=True + ) + + processed_commit_info = [] + raw_commits = raw_commit_info.stdout.splitlines() + for commit in raw_commits: + split_fields = commit.split("^_^") + processed_commit_info.append( + { + "report_flag": "", + "date": split_fields[0], + "sha": split_fields[1], + "author": split_fields[2], + "message": split_fields[3], + } + ) + add_report_message_to_git_commits(processed_commit_info) + return processed_commit_info + + def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): """ Create the subheader for the HTML report. @@ -163,7 +299,9 @@ def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): return subheader -def render_html_report(report_data, subheader): +def render_html_report( + report_data, subheader, esmval_core_commits, esmval_tool_commits, + ): """ Render the HTML report using Jinja2. @@ -173,6 +311,10 @@ def render_html_report(report_data, subheader): The report data to be rendered in the HTML template. subheader : str The subheader for the HTML report. + esmval_core_commits : dict + The ESMValCore commits information. + esmval_tool_commits : dict + The ESMValTool commits information. Returns ------- @@ -188,6 +330,8 @@ def render_html_report(report_data, subheader): rendered_html = template.render( subheader=subheader, report_data=report_data, + esmval_core_commits=esmval_core_commits, + esmval_tool_commits=esmval_tool_commits, ) return rendered_html diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index a869007959..fb43456da1 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -28,10 +28,67 @@ color: white; } + +

Recipe Test Workflow - Last Run Status

+

{{ subheader }}

+ + {% if esmval_core_commits %} + + + + + + + + + + {% for commit in esmval_core_commits %} + + {% if commit['report_flag'] %} + + {% else %} + + {% endif %} + + + + + + {% endfor %} +

ESMValCore Commits

TimeSHAAuthorCommit message
{{ commit['report_flag'] }}{{ commit['date'] }}{{ commit['sha'] }}{{ commit['author'] }} {{ commit['message']}}
+ {% endif %} + + {% if esmval_tool_commits %} + + + + + + + + + + {% for commit in esmval_tool_commits %} + + {% if commit['report_flag'] %} + + {% else %} + + {% endif %} + + + + + + {% endfor %} +

ESMValTool Commits

TimeSHAAuthorCommit message
{{ commit['report_flag'] }}{{ commit['date'] }}{{ commit['sha'] }}{{ commit['author'] }} {{ commit['message']}}
+ +
+ {% endif %} + -

Recipe Test Workflow - Last Run Status

-

{{ subheader }}

+ From 3ae7fa96d331264fb3f0dc6bb08401a118b52d20 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 09:25:55 +0100 Subject: [PATCH 04/40] #4036: Unindent line --- .../app/generate_report/bin/generate_html_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index fdda333232..b44451cc9d 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -275,7 +275,7 @@ def fetch_git_commits(package_path, sha=None): "message": split_fields[3], } ) - add_report_message_to_git_commits(processed_commit_info) + add_report_message_to_git_commits(processed_commit_info) return processed_commit_info From 7ee9dce37612265dace8a8cacc9ac75a33e575f0 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 12:29:23 +0100 Subject: [PATCH 05/40] #4036: Import DKRZ container vars. Test subprocess cmd --- .../bin/generate_html_report.py | 84 ++++++++++++------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index b44451cc9d..d02873f068 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -36,18 +36,22 @@ # ] -# UNCOMMENT FOR REAL +# UNCOMMENT FOR MET OFFICE / DKRZ CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") CYLC_TASK_PREVIOUS_CYCLE = os.environ.get("ROSE_DATACP1D") REPORT_PATH = os.environ.get("REPORT_PATH") -ESMVAL_CORE_CURRENT = os.environ.get("ESMVALCORE_DIR") -ESMVAL_TOOL_CURRENT = os.environ.get("ESMVALTOOL_DIR") - -ESMVAL_CORE_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValCore" -ESMVAL_TOOL_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValTool" +# UNCOMMENT FOR MET OFFICE +# ESMVAL_CORE_CURRENT = os.environ.get("ESMVALCORE_DIR") +# ESMVAL_TOOL_CURRENT = os.environ.get("ESMVALTOOL_DIR") +# ESMVAL_CORE_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValCore" +# ESMVAL_TOOL_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValTool" +# UNCOMMENT FOR DKRZ +CONTAINER_DIR = os.environ.get("CONTAINER_DIR") +CONTAINER_FILE = "esmvaltool.sif" +CONTAINER_PATH = os.environ.get("CONTAINER_PATH") SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" @@ -61,7 +65,6 @@ def main(db_file_path=CYLC_DB_PATH): db_file_path : str, default CYLC_DB_FILE_PATH The path to the SQLite database file. """ - # UNCOMMENT FOR LOCAL # raw_db_data = cached_raw_db_data # processed_db_data = process_db_output(raw_db_data) @@ -76,35 +79,42 @@ def main(db_file_path=CYLC_DB_PATH): # local_esmvaltool, esmval_tool_previous_commit_sha # ) - # UNCOMMENT FOR REAL - raw_db_data = fetch_report_data(db_file_path) - processed_db_data = process_db_output(raw_db_data) + # UNCOMMENT FOR MET OFFICE + # raw_db_data = fetch_report_data(db_file_path) + # processed_db_data = process_db_output(raw_db_data) - esmval_core_previous_commit_sha = None - if ESMVAL_CORE_PREVIOUS.exists(): - esmval_core_previous_commit_sha = ( - fetch_git_commits(ESMVAL_CORE_PREVIOUS)['sha'] - ) + # esmval_core_previous_commit_sha = None + # if ESMVAL_CORE_PREVIOUS.exists(): + # esmval_core_previous_commit_sha = ( + # fetch_git_commits(ESMVAL_CORE_PREVIOUS)['sha'] + # ) - esmval_tool_previous_commit_sha = None - if ESMVAL_TOOL_PREVIOUS.exists(): - esmval_tool_previous_commit_sha = ( - fetch_git_commits(ESMVAL_TOOL_PREVIOUS)['sha'] - ) + # esmval_tool_previous_commit_sha = None + # if ESMVAL_TOOL_PREVIOUS.exists(): + # esmval_tool_previous_commit_sha = ( + # fetch_git_commits(ESMVAL_TOOL_PREVIOUS)['sha'] + # ) - esmval_core_all_commits = fetch_git_commits( - ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha - ) - esmval_tool_all_commits = fetch_git_commits( - ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha - ) + # esmval_core_all_commits = fetch_git_commits( + # ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha + # ) + # esmval_tool_all_commits = fetch_git_commits( + # ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha + # ) + print("Container dir: ", CONTAINER_DIR) + print("Container path: ", CONTAINER_PATH) + print("Previous cylce point: ", CYLC_TASK_PREVIOUS_CYCLE) + + raw_db_data = fetch_report_data(db_file_path) + processed_db_data = process_db_output(raw_db_data) + current_package_versions = "" subheader = create_subheader() rendered_html = render_html_report( subheader=subheader, report_data=processed_db_data, - esmval_core_commits=esmval_core_all_commits, - esmval_tool_commits=esmval_tool_all_commits, + # esmval_core_commits=esmval_core_all_commits, + # esmval_tool_commits=esmval_tool_all_commits, ) write_report_to_file(rendered_html) @@ -219,6 +229,20 @@ def process_db_output(report_data): return sorted_processed_db_data +def fetch_package_versions_from_container(path_to_container): + command = [path_to_container, "esmvaltool", "version"] + print(command) + raw_version_info = subprocess.run( + command, + capture_output=True, + check=True, + text=True + ) + print(command.stdout) + print(command.stderr) + return command.stdout + + def add_report_message_to_git_commits(git_commits_info): """ Add report messages to a git commit information dictionary. @@ -228,9 +252,9 @@ def add_report_message_to_git_commits(git_commits_info): list[dict] A list of git commits. """ - git_commits_info[0]['report_flag'] = "Version tested >>>" + git_commits_info[0]['report_flag'] = "Version tested this cycle >>>" if len(git_commits_info) > 1: - git_commits_info[-1]['report_flag'] = "Version last tested >>>" + git_commits_info[-1]['report_flag'] = "Version tested last cycle >>>" def fetch_git_commits(package_path, sha=None): From 529f350ab943f0db3e8d11b05c14bbbe16bfa88c Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 12:45:56 +0100 Subject: [PATCH 06/40] #4036: Run fetch_packages cmd on dkrz --- .../app/generate_report/bin/generate_html_report.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index d02873f068..b860d4cb6f 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -108,7 +108,11 @@ def main(db_file_path=CYLC_DB_PATH): raw_db_data = fetch_report_data(db_file_path) processed_db_data = process_db_output(raw_db_data) - current_package_versions = "" + + print("Fetching package versions") + current_package_versions = fetch_package_versions_from_container() + print("Package versions fetched") + subheader = create_subheader() rendered_html = render_html_report( subheader=subheader, From 57b55fdc2f57ec471d4ff11cb60db4a8199b3907 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 12:58:50 +0100 Subject: [PATCH 07/40] #4036: Correct func call --- .../app/generate_report/bin/generate_html_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index b860d4cb6f..6dd69631dc 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -110,7 +110,7 @@ def main(db_file_path=CYLC_DB_PATH): processed_db_data = process_db_output(raw_db_data) print("Fetching package versions") - current_package_versions = fetch_package_versions_from_container() + current_package_versions = fetch_package_versions_from_container(CONTAINER_PATH) print("Package versions fetched") subheader = create_subheader() From 237b81492a051fd89a353ee38a1892437856e11a Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 13:14:06 +0100 Subject: [PATCH 08/40] #4036: Pass site env file into script --- .../app/generate_report/bin/generate_html_report.py | 3 ++- esmvaltool/utils/recipe_test_workflow/flow.cylc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 6dd69631dc..973ebaf2b6 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -52,6 +52,7 @@ CONTAINER_DIR = os.environ.get("CONTAINER_DIR") CONTAINER_FILE = "esmvaltool.sif" CONTAINER_PATH = os.environ.get("CONTAINER_PATH") +ENV_FILE = os.environ.get("ENV_FILE_SITE_PATH") SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" @@ -234,7 +235,7 @@ def process_db_output(report_data): def fetch_package_versions_from_container(path_to_container): - command = [path_to_container, "esmvaltool", "version"] + command = [ENV_FILE, path_to_container, "esmvaltool", "version"] print(command) raw_version_info = subprocess.run( command, diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index 8a41d9c06b..19fdd3c852 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -160,6 +160,7 @@ CYLC_DB_PATH = ${CYLC_WORKFLOW_RUN_DIR}/log/db PRODUCTION = {{ PRODUCTION }} REPORT_PATH = ${ROSE_DATAC}/status_report.html + ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}/env-file [[housekeeping]] platform = localhost From 8e221af71603c2c3c5802256060c1ffd28900980 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 13:27:40 +0100 Subject: [PATCH 09/40] #4036: Use site env in command --- esmvaltool/utils/recipe_test_workflow/flow.cylc | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index 19fdd3c852..61e7ac2758 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -160,6 +160,7 @@ CYLC_DB_PATH = ${CYLC_WORKFLOW_RUN_DIR}/log/db PRODUCTION = {{ PRODUCTION }} REPORT_PATH = ${ROSE_DATAC}/status_report.html + SITE = {{ SITE }} ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}/env-file [[housekeeping]] From e30045597062665208397fc00006af8a97f47ae9 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 13:46:35 +0100 Subject: [PATCH 10/40] #4036: Tweak command --- .../app/generate_report/bin/generate_html_report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 973ebaf2b6..9ea1e07b4e 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -111,7 +111,7 @@ def main(db_file_path=CYLC_DB_PATH): processed_db_data = process_db_output(raw_db_data) print("Fetching package versions") - current_package_versions = fetch_package_versions_from_container(CONTAINER_PATH) + current_package_versions = fetch_package_versions_from_container() print("Package versions fetched") subheader = create_subheader() @@ -234,8 +234,8 @@ def process_db_output(report_data): return sorted_processed_db_data -def fetch_package_versions_from_container(path_to_container): - command = [ENV_FILE, path_to_container, "esmvaltool", "version"] +def fetch_package_versions_from_container(): + command = [ENV_FILE, "esmvaltool", "version"] print(command) raw_version_info = subprocess.run( command, From 290ef02cfb05ef4d0f2e0af140a71714ef82b61b Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 14:06:09 +0100 Subject: [PATCH 11/40] #4036: Update env-file path --- .../app/generate_report/bin/generate_html_report.py | 2 +- esmvaltool/utils/recipe_test_workflow/flow.cylc | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 9ea1e07b4e..c5949291fb 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -52,7 +52,7 @@ CONTAINER_DIR = os.environ.get("CONTAINER_DIR") CONTAINER_FILE = "esmvaltool.sif" CONTAINER_PATH = os.environ.get("CONTAINER_PATH") -ENV_FILE = os.environ.get("ENV_FILE_SITE_PATH") +ENV_FILE = os.environ.get("ENV_FILE") SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index 61e7ac2758..a323acc639 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -160,8 +160,7 @@ CYLC_DB_PATH = ${CYLC_WORKFLOW_RUN_DIR}/log/db PRODUCTION = {{ PRODUCTION }} REPORT_PATH = ${ROSE_DATAC}/status_report.html - SITE = {{ SITE }} - ENV_FILE_SITE_PATH=${CYLC_WORKFLOW_RUN_DIR}/site/${SITE}/env-file + ENV_FILE = ${CYLC_WORKFLOW_SHARE_DIR}/bin/env-file [[housekeeping]] platform = localhost From bc6163b3683fcc921fe1b2f6ed9d03a7cf082e64 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 14:48:21 +0100 Subject: [PATCH 12/40] #4036: Call command in rose conf. Call with sing env in python --- .../bin/generate_html_report.py | 28 ++++++++++++++----- .../app/generate_report/rose-app.conf | 3 +- .../utils/recipe_test_workflow/flow.cylc | 4 ++- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index c5949291fb..42c5d8bf60 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -50,9 +50,12 @@ # UNCOMMENT FOR DKRZ CONTAINER_DIR = os.environ.get("CONTAINER_DIR") -CONTAINER_FILE = "esmvaltool.sif" +CONTAINER = "esmvaltool.sif" CONTAINER_PATH = os.environ.get("CONTAINER_PATH") + +SHARE_BIN = os.environ.get("SHARE_BIN") ENV_FILE = os.environ.get("ENV_FILE") +SING_ENV_FILE = os.environ.get("SING_ENV_FILE") SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" @@ -105,7 +108,16 @@ def main(db_file_path=CYLC_DB_PATH): print("Container dir: ", CONTAINER_DIR) print("Container path: ", CONTAINER_PATH) - print("Previous cylce point: ", CYLC_TASK_PREVIOUS_CYCLE) + print("Previous cycle point: ", CYLC_TASK_PREVIOUS_CYCLE) + print("Share bin: ", SHARE_BIN) + print("Env file: ", ENV_FILE) + print("Singularity env: ", SING_ENV_FILE) + + print("Share bin exists?: ", Path(SHARE_BIN).exists()) + print("Env file exists?: ", Path(ENV_FILE).exists()) + print("Sing env exists?", Path(SING_ENV_FILE).exists()) + + print("Env file content", Path(ENV_FILE).read_text()) raw_db_data = fetch_report_data(db_file_path) processed_db_data = process_db_output(raw_db_data) @@ -235,17 +247,19 @@ def process_db_output(report_data): def fetch_package_versions_from_container(): - command = [ENV_FILE, "esmvaltool", "version"] - print(command) + print("Cwd", CONTAINER_DIR) + command = [SING_ENV_FILE, "singularity", "run", "esmvaltool.sif", "version"] + print("Command: ", command) raw_version_info = subprocess.run( command, + cwd=CONTAINER_DIR, capture_output=True, check=True, text=True ) - print(command.stdout) - print(command.stderr) - return command.stdout + print(raw_version_info.stdout) + print(raw_version_info.stderr) + return raw_version_info.stdout def add_report_message_to_git_commits(git_commits_info): diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf index cf3db61d85..07f4efec74 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf @@ -1,2 +1,3 @@ [command] -default=env-file generate_html_report.py +default=env-file esmvaltool version + =env-file generate_html_report.py diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index a323acc639..f39c4f4655 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -160,7 +160,9 @@ CYLC_DB_PATH = ${CYLC_WORKFLOW_RUN_DIR}/log/db PRODUCTION = {{ PRODUCTION }} REPORT_PATH = ${ROSE_DATAC}/status_report.html - ENV_FILE = ${CYLC_WORKFLOW_SHARE_DIR}/bin/env-file + SHARE_BIN = ${CYLC_WORKFLOW_SHARE_DIR}/bin + ENV_FILE = ${SHARE_BIN}/env-file + SING_ENV_FILE = ${SHARE_BIN}/singularity-env-file [[housekeeping]] platform = localhost From 78b2d163adf99702322bbe800071f0b47f96176f Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 21 May 2025 15:01:34 +0100 Subject: [PATCH 13/40] #4036: Run only version command in rose-app.conf --- .../recipe_test_workflow/app/generate_report/rose-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf index 07f4efec74..b01208c50b 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf @@ -1,3 +1,3 @@ [command] default=env-file esmvaltool version - =env-file generate_html_report.py + From bdcfae2e3b486d2e051a207003a39803288a8cb0 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Tue, 27 May 2025 15:55:57 +0100 Subject: [PATCH 14/40] #4036: Make generate report multisite. Tweak env files. DKRZ get versions in rose-app.conf --- .../bin/generate_html_report.py | 155 ++++++++---------- .../generate_report/opt/rose-app-dkrz.conf | 2 + .../app/generate_report/rose-app.conf | 4 +- .../utils/recipe_test_workflow/flow.cylc | 1 + .../recipe_test_workflow/site/dkrz/env-file | 11 +- .../site/metoffice/env-file | 11 +- 6 files changed, 93 insertions(+), 91 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 42c5d8bf60..dcbd5dc5ed 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -1,9 +1,9 @@ #!/usr/bin/env python import os import sqlite3 +import subprocess from datetime import datetime from pathlib import Path -import subprocess from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -35,27 +35,25 @@ # ('compare_recipe_ocean_amoc','succeeded') # ] - -# UNCOMMENT FOR MET OFFICE / DKRZ +# Load environment variables required at all sites. CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") CYLC_TASK_PREVIOUS_CYCLE = os.environ.get("ROSE_DATACP1D") REPORT_PATH = os.environ.get("REPORT_PATH") +SITE = os.environ.get("SITE") -# UNCOMMENT FOR MET OFFICE -# ESMVAL_CORE_CURRENT = os.environ.get("ESMVALCORE_DIR") -# ESMVAL_TOOL_CURRENT = os.environ.get("ESMVALTOOL_DIR") -# ESMVAL_CORE_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValCore" -# ESMVAL_TOOL_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValTool" - -# UNCOMMENT FOR DKRZ -CONTAINER_DIR = os.environ.get("CONTAINER_DIR") -CONTAINER = "esmvaltool.sif" -CONTAINER_PATH = os.environ.get("CONTAINER_PATH") +if SITE == "metoffice": + # Load Met Office specific environment variables. + ESMVAL_CORE_CURRENT = os.environ.get("ESMVALCORE_DIR") + ESMVAL_TOOL_CURRENT = os.environ.get("ESMVALTOOL_DIR") + ESMVAL_CORE_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValCore" + ESMVAL_TOOL_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValTool" + ESMVAL_VERSIONS = os.environ.get("ESMVAL_VERSIONS") -SHARE_BIN = os.environ.get("SHARE_BIN") -ENV_FILE = os.environ.get("ENV_FILE") -SING_ENV_FILE = os.environ.get("SING_ENV_FILE") +elif SITE == "dkrz": + # Load DKRZ specific environment variables. + ESMVAL_VERSIONS_CURRENT = os.environ.get("ESMVAL_VERSIONS_CURRENT") + ESMVAL_VERSIONS_PREVIOUS = os.environ.get("ESMVAL_VERSIONS_PREVIOUS") SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" @@ -72,67 +70,63 @@ def main(db_file_path=CYLC_DB_PATH): # UNCOMMENT FOR LOCAL # raw_db_data = cached_raw_db_data # processed_db_data = process_db_output(raw_db_data) - # local_esmvaltool = Path("/home/users/christopher.billows/Code/ESMValTool/") - # local_esmvalcore = Path("/home/users/christopher.billows/Code/ESMValCore/") + # ESMVAL_TOOL_CURRENT = Path("/home/users/christopher.billows/Code/ESMValTool/") + # ESMVAL_CORE_CURRENT = Path("/home/users/christopher.billows/Code/ESMValCore/") # esmval_core_previous_commit_sha = "170a93893" # esmval_tool_previous_commit_sha = "4515a2b92" - # esmval_core_all_commits = fetch_git_commits( - # local_esmvalcore, esmval_core_previous_commit_sha - # ) - # esmval_tool_all_commits = fetch_git_commits( - # local_esmvaltool, esmval_tool_previous_commit_sha - # ) - - # UNCOMMENT FOR MET OFFICE - # raw_db_data = fetch_report_data(db_file_path) - # processed_db_data = process_db_output(raw_db_data) - - # esmval_core_previous_commit_sha = None - # if ESMVAL_CORE_PREVIOUS.exists(): - # esmval_core_previous_commit_sha = ( - # fetch_git_commits(ESMVAL_CORE_PREVIOUS)['sha'] - # ) - - # esmval_tool_previous_commit_sha = None - # if ESMVAL_TOOL_PREVIOUS.exists(): - # esmval_tool_previous_commit_sha = ( - # fetch_git_commits(ESMVAL_TOOL_PREVIOUS)['sha'] - # ) - # esmval_core_all_commits = fetch_git_commits( # ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha # ) # esmval_tool_all_commits = fetch_git_commits( # ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha # ) + # SITE="metoffice" + # ESMVAL_CORE_PREVIOUS = Path("nope") + # ESMVAL_TOOL_PREVIOUS = ESMVAL_TOOL_CURRENT - print("Container dir: ", CONTAINER_DIR) - print("Container path: ", CONTAINER_PATH) - print("Previous cycle point: ", CYLC_TASK_PREVIOUS_CYCLE) - print("Share bin: ", SHARE_BIN) - print("Env file: ", ENV_FILE) - print("Singularity env: ", SING_ENV_FILE) + raw_db_data = fetch_report_data(db_file_path) + processed_db_data = process_db_output(raw_db_data) - print("Share bin exists?: ", Path(SHARE_BIN).exists()) - print("Env file exists?: ", Path(ENV_FILE).exists()) - print("Sing env exists?", Path(SING_ENV_FILE).exists()) + if SITE == "metoffice": + if ESMVAL_CORE_PREVIOUS.exists(): + esmval_core_previous_commit_sha = fetch_git_commits( + ESMVAL_CORE_PREVIOUS + )[0]["sha"] + else: + esmval_core_previous_commit_sha = None + + if ESMVAL_TOOL_PREVIOUS.exists(): + esmval_tool_previous_commit_sha = fetch_git_commits( + ESMVAL_TOOL_PREVIOUS + )[0]["sha"] + else: + esmval_core_previous_commit_sha = None + + esmval_core_all_commits = fetch_git_commits( + ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha + ) + esmval_tool_all_commits = fetch_git_commits( + ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha + ) - print("Env file content", Path(ENV_FILE).read_text()) + subheader = create_subheader() + rendered_html = render_html_report( + subheader=subheader, + report_data=processed_db_data, + esmval_core_commits=esmval_core_all_commits, + esmval_tool_commits=esmval_tool_all_commits, + ) - raw_db_data = fetch_report_data(db_file_path) - processed_db_data = process_db_output(raw_db_data) + elif SITE == "dkrz": + print("Current versons in Python", ESMVAL_VERSIONS_CURRENT) - print("Fetching package versions") - current_package_versions = fetch_package_versions_from_container() - print("Package versions fetched") + else: + subheader = create_subheader() + rendered_html = render_html_report( + subheader=subheader, + report_data=processed_db_data, + ) - subheader = create_subheader() - rendered_html = render_html_report( - subheader=subheader, - report_data=processed_db_data, - # esmval_core_commits=esmval_core_all_commits, - # esmval_tool_commits=esmval_tool_all_commits, - ) write_report_to_file(rendered_html) @@ -246,22 +240,6 @@ def process_db_output(report_data): return sorted_processed_db_data -def fetch_package_versions_from_container(): - print("Cwd", CONTAINER_DIR) - command = [SING_ENV_FILE, "singularity", "run", "esmvaltool.sif", "version"] - print("Command: ", command) - raw_version_info = subprocess.run( - command, - cwd=CONTAINER_DIR, - capture_output=True, - check=True, - text=True - ) - print(raw_version_info.stdout) - print(raw_version_info.stderr) - return raw_version_info.stdout - - def add_report_message_to_git_commits(git_commits_info): """ Add report messages to a git commit information dictionary. @@ -271,9 +249,9 @@ def add_report_message_to_git_commits(git_commits_info): list[dict] A list of git commits. """ - git_commits_info[0]['report_flag'] = "Version tested this cycle >>>" + git_commits_info[0]["report_flag"] = "Version tested this cycle >>>" if len(git_commits_info) > 1: - git_commits_info[-1]['report_flag'] = "Version tested last cycle >>>" + git_commits_info[-1]["report_flag"] = "Version tested last cycle >>>" def fetch_git_commits(package_path, sha=None): @@ -295,8 +273,12 @@ def fetch_git_commits(package_path, sha=None): passed, multiple commits/dicts may be returned. """ command = [ - "git", "log", "-1", "--date=iso-strict", "--pretty=%cd^_^%h^_^%an^_^%s" - ] + "git", + "log", + "-1", + "--date=iso-strict", + "--pretty=%cd^_^%h^_^%an^_^%s", + ] if sha: command[2] = f"{sha}^..HEAD" @@ -343,8 +325,11 @@ def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): def render_html_report( - report_data, subheader, esmval_core_commits, esmval_tool_commits, - ): + report_data, + subheader, + esmval_core_commits, + esmval_tool_commits, +): """ Render the HTML report using Jinja2. diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf index 85bd994877..f229d01ac3 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf @@ -1,5 +1,7 @@ [command] default=set -euo pipefail + =export ESMVAL_VERSIONS_CURRENT=$(CAPTURE_OUTPUT=True env-file esmvaltool version) + =echo $ESMVAL_VERSIONS_CURRENT =env-file generate_html_report.py =if [ "${PRODUCTION}" = "True" ]; then rsync -av "${REPORT_PATH}" "${VM_PATH}"; echo "HTML report copied from ${REPORT_PATH} to ${VM_PATH}"; fi diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf index b01208c50b..b62e4505bb 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf @@ -1,3 +1,3 @@ [command] -default=env-file esmvaltool version - +default=export ESMVAL_VERSIONS=$(QUIET_MODE=True env-file esmvaltool version) + =env-file generate_html_report.py diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index f39c4f4655..8da114b1a5 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -156,6 +156,7 @@ [[[environment]]] # By default opt configurations must exist. Paretheses make the opt # file optional, which is required as only a DKRZ opt file exists. + SITE = {{ SITE }} ROSE_APP_OPT_CONF_KEYS = ({{ SITE }}) CYLC_DB_PATH = ${CYLC_WORKFLOW_RUN_DIR}/log/db PRODUCTION = {{ PRODUCTION }} diff --git a/esmvaltool/utils/recipe_test_workflow/site/dkrz/env-file b/esmvaltool/utils/recipe_test_workflow/site/dkrz/env-file index 0ae73c73cb..c8978cb882 100755 --- a/esmvaltool/utils/recipe_test_workflow/site/dkrz/env-file +++ b/esmvaltool/utils/recipe_test_workflow/site/dkrz/env-file @@ -49,8 +49,10 @@ export SINGULARITYENV_APPEND_PATH="${WORKFLOW_RUN_BIN_DIR}:${WORKFLOW_SHARE_BIN_ # If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the # Python environment. if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then - echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}" export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-} + if [[ -z ${QUIET_MODE:-} ]]; then + echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}" + fi fi if [[ -z ${QUIET_MODE:-} ]]; then @@ -59,4 +61,9 @@ fi singularity_command="singularity-env-file singularity -q exec ${CONTAINER_PATH} $@" command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time ${singularity_command}" -exec ${command} + +if [[ -n "${CAPTURE_OUTPUT:-}" ]]; then + ${command} +else + exec ${command} +fi diff --git a/esmvaltool/utils/recipe_test_workflow/site/metoffice/env-file b/esmvaltool/utils/recipe_test_workflow/site/metoffice/env-file index 79e09625ea..64bf1f2867 100755 --- a/esmvaltool/utils/recipe_test_workflow/site/metoffice/env-file +++ b/esmvaltool/utils/recipe_test_workflow/site/metoffice/env-file @@ -48,8 +48,10 @@ safe_load "${ENV_NAME}" # If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the # Python environment. if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then - echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}" export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-} + if [[ -z ${QUIET_MODE:-} ]]; then + echo "[INFO] Prepending the following to PYTHONPATH: ${PYTHONPATH_PREPEND}" + fi fi if [[ -z ${QUIET_MODE:-} ]]; then @@ -57,4 +59,9 @@ if [[ -z ${QUIET_MODE:-} ]]; then fi command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time $@" -exec ${command} + +if [[ -n "${CAPTURE_OUTPUT:-}" ]]; then + ${command} +else + exec ${command} +fi From d10ddc80d2d100c053c50c21798c797218805970 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Tue, 27 May 2025 16:00:34 +0100 Subject: [PATCH 15/40] #4036: Run rose config dump --- .../app/get_esmval/opt/rose-app-metoffice.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf index 4a6a5c3472..5c96bdd5d6 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/get_esmval/opt/rose-app-metoffice.conf @@ -7,9 +7,9 @@ ESMVALCORE_URL=git@github.com:ESMValGroup/ESMValCore.git ESMVALTOOL_URL=git@github.com:ESMValGroup/ESMValTool.git [file:${CYLC_WORKFLOW_SHARE_DIR}/lib/python/ESMValCore] -source=${ROSE_DATAC}/ESMValCore/ mode=symlink +source=${ROSE_DATAC}/ESMValCore/ [file:${CYLC_WORKFLOW_SHARE_DIR}/lib/python/ESMValTool] -source=${ROSE_DATAC}/ESMValTool/ mode=symlink +source=${ROSE_DATAC}/ESMValTool/ From dbb0557566f0cd4f37d2791eb1590e4ae27c4303 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 28 May 2025 10:47:49 +0100 Subject: [PATCH 16/40] #4036: Remove development/local version --- .../bin/generate_html_report.py | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index dcbd5dc5ed..ccbd4add6d 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -7,34 +7,6 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape -# UNCOMMENT FOR LOCAL -# CYLC_DB_PATH = "hello" -# CYLC_TASK_CYCLE_POINT = "20250516T1053Z" -# REPORT_PATH="/home/users/christopher.billows/Code/ESMValTool/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report.html" -# cached_raw_db_data = [ -# ('get_esmval', 'waiting'), -# ('install_env_file','succeeded'), -# ('get_esmval','succeeded'), -# ('configure','succeeded'), -# ('compare_recipe_radiation_budget','succeeded'), -# ('process_recipe_radiation_budget','succeeded'), -# ('process_recipe_albedolandcover','succeeded'), -# ('process_recipe_ocean_amoc','succeeded'), -# ('process_recipe_autoassess_landsurface_soilmoisture','succeeded'), -# ('process_recipe_heatwaves_coldwaves','succeeded'), -# ('process_recipe_ocean_multimap','succeeded'), -# ('process_recipe_ensclus','succeeded'), -# ('process_recipe_consecdrydays','succeeded'), -# ('compare_recipe_albedolandcover','succeeded'), -# ('compare_recipe_consecdrydays','succeeded'), -# ('generate_report','running'), -# ('compare_recipe_autoassess_landsurface_soilmoisture','succeeded'), -# ('compare_recipe_heatwaves_coldwaves','succeeded'), -# ('compare_recipe_ensclus','succeeded'), -# ('compare_recipe_ocean_multimap','succeeded'), -# ('compare_recipe_ocean_amoc','succeeded') -# ] - # Load environment variables required at all sites. CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") @@ -67,23 +39,6 @@ def main(db_file_path=CYLC_DB_PATH): db_file_path : str, default CYLC_DB_FILE_PATH The path to the SQLite database file. """ - # UNCOMMENT FOR LOCAL - # raw_db_data = cached_raw_db_data - # processed_db_data = process_db_output(raw_db_data) - # ESMVAL_TOOL_CURRENT = Path("/home/users/christopher.billows/Code/ESMValTool/") - # ESMVAL_CORE_CURRENT = Path("/home/users/christopher.billows/Code/ESMValCore/") - # esmval_core_previous_commit_sha = "170a93893" - # esmval_tool_previous_commit_sha = "4515a2b92" - # esmval_core_all_commits = fetch_git_commits( - # ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha - # ) - # esmval_tool_all_commits = fetch_git_commits( - # ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha - # ) - # SITE="metoffice" - # ESMVAL_CORE_PREVIOUS = Path("nope") - # ESMVAL_TOOL_PREVIOUS = ESMVAL_TOOL_CURRENT - raw_db_data = fetch_report_data(db_file_path) processed_db_data = process_db_output(raw_db_data) From aad0813f1c940dee0860736c83b6ab9bccd7a40a Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Fri, 30 May 2025 12:23:23 +0100 Subject: [PATCH 17/40] #4036: WIP commit. Split sha/commit fetching into modules. Refactor logic --- .../generate_report/bin/commits_via_git.py | 151 ++++++++++++++ .../bin/generate_html_report.py | 190 ++++++------------ .../bin/sha_via_singularity.py | 97 +++++++++ .../bin/test_generate_html_report.py | 38 ++-- .../bin/test_sha_via_singularity.py | 31 +++ 5 files changed, 362 insertions(+), 145 deletions(-) create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/sha_via_singularity.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py new file mode 100644 index 0000000000..32539bd98f --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py @@ -0,0 +1,151 @@ +""" +Functions to fetch commit information from local git repositories. +""" + +import subprocess +from pathlib import Path + + +def get_commits_from_git(repos): + """ + Fetch commit information from local git repos. + + Parameters + ---------- + repos : dict[str, str | None] + A dictionary where keys are in the form ``repo_day`` and values are + the path to the repo, or None. E.g. ``{"core_today": "path/to/repo", + "core_yesterday": None}`` + + Raises + ------ + ValueError + If repos does not contain enough valid git directories to fetch + useable commit data. + + Returns + ------- + tuple[list[dict], list[dict]] + A tuple of two lists of dictionaries. Each dictionary include + information on a commit or a range of commits for + ``(ESMValCore, ESMValTool)``. Each commit dict has the following fields + ``date, sha, author, message``. + """ + repo_validity = {key: is_git_repo(path) for key, path in repos.items()} + + if not any(repo_validity.values()): + raise ValueError("No valid git repos passed") + + elif not (repo_validity["core_today"] or repo_validity["tool_today"]): + raise ValueError("Today's git commits unavailable.") + + elif not ( + repo_validity["core_yesterday"] and repo_validity["tool_yesterday"] + ): + print("Only today's git info is available.") + commit_info = ( + query_git_log(repos["core_today"]), + query_git_log(repos["tool_today"]), + ) + + else: + commit_info = get_all_commits_for_today_and_yesterday(repos) + + return commit_info + + +def is_git_repo(path): + """ + Check a passed value is a valid git directory. + + Parameters + ---------- + path : str|None + Path to a git repo, or None. + + Returns + ------- + bool + If the passed value is a valid git directory. + """ + try: + return Path(path).expanduser().resolve().joinpath(".git").is_dir() + except (TypeError, OSError): + return False + + +def get_all_commits_for_today_and_yesterday(valid_repos): + """ + Fetch information on a range of commits. + + Fetches a range of commits + + Parameters + ---------- + valid_repos : dict[str, str] + A dict of valid git repos. + + Returns + ------- + tuple[list[dict], list[dict]] + A tuple with two lists of dictionaries that include a range of commit + information for ``(ESMValCore, ESMValTool)``. + + """ + core_yesterday_sha = query_git_log(valid_repos["core_yesterday"][0]["sha"]) + core_commits = query_git_log(valid_repos["core_today"], core_yesterday_sha) + + tool_yesterday_sha = query_git_log(valid_repos["tool_yesterday"][0]["sha"]) + tool_commits = query_git_log(valid_repos["tool_today"], tool_yesterday_sha) + + return (core_commits, tool_commits) + + +def query_git_log(package_path, sha=None): + """ + Use ``git log`` to fetch commit information from a local git repo. + + Parameters + ---------- + package_path : str + Path to a valid git repo. + sha: str | None + Optional. The sha of a previously tested commit. If provided, commits + from HEAD back to the passed sha (inclusive) will be retrieved. + + Returns + ------- + list[dict] + A list of dicts where each dict represents one commit. Each commit has + the following fields ``date, sha, author, message``. If ``sha`` is + not passed, one commit is returned. If ``sha`` is passed, a minimum of + two commits is returned. + """ + command = [ + "git", + "log", + "-1", + "--date=iso-strict", + "--pretty=%cd^_^%h^_^%an^_^%s", + ] + + if sha: + command[2] = f"{sha}^..HEAD" + + raw_commit_info = subprocess.run( + command, cwd=package_path, capture_output=True, check=True, text=True + ) + processed_commit_info = [] + raw_commits = raw_commit_info.stdout.splitlines() + for commit in raw_commits: + split_fields = commit.split("^_^") + processed_commit_info.append( + { + "report_flag": "", + "date": split_fields[0], + "sha": split_fields[1], + "author": split_fields[2], + "message": split_fields[3], + } + ) + return processed_commit_info diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index ccbd4add6d..208bed4c67 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -1,36 +1,38 @@ #!/usr/bin/env python import os import sqlite3 -import subprocess from datetime import datetime from pathlib import Path from jinja2 import Environment, FileSystemLoader, select_autoescape +from bin.commits_via_git import get_commits_from_git +from bin.sha_via_singularity import get_shas_from_singularity + # Load environment variables required at all sites. CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") -CYLC_TASK_PREVIOUS_CYCLE = os.environ.get("ROSE_DATACP1D") +CYLC_TASK_CYCLE_YESTERDAY = os.environ.get("ROSE_DATACP1D") REPORT_PATH = os.environ.get("REPORT_PATH") SITE = os.environ.get("SITE") -if SITE == "metoffice": - # Load Met Office specific environment variables. - ESMVAL_CORE_CURRENT = os.environ.get("ESMVALCORE_DIR") - ESMVAL_TOOL_CURRENT = os.environ.get("ESMVALTOOL_DIR") - ESMVAL_CORE_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValCore" - ESMVAL_TOOL_PREVIOUS = Path(CYLC_TASK_PREVIOUS_CYCLE) / "ESMValTool" - ESMVAL_VERSIONS = os.environ.get("ESMVAL_VERSIONS") +if SITE == "dkrz": + ESMVAL_VERSIONS_TODAY = os.environ.get("ESMVAL_VERSIONS_CURRENT") + ESMVAL_VERSIONS_YESTERDAY = os.environ.get("ESMVAL_VERSIONS_PREVIOUS") -elif SITE == "dkrz": - # Load DKRZ specific environment variables. - ESMVAL_VERSIONS_CURRENT = os.environ.get("ESMVAL_VERSIONS_CURRENT") - ESMVAL_VERSIONS_PREVIOUS = os.environ.get("ESMVAL_VERSIONS_PREVIOUS") +elif SITE == "metoffice": + REPOS = { + "core_today": os.environ.get("ESMVALCORE_DIR"), + "tool_today": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValCore", + "core_yesterday": os.environ.get("ESMVALTOOL_DIR"), + "tool_yesterday": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValTool", + } + print("Repos", REPOS) SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" -def main(db_file_path=CYLC_DB_PATH): +def main(db_file_path=CYLC_DB_PATH, site=SITE): """ Main function to generate the HTML report. @@ -41,47 +43,37 @@ def main(db_file_path=CYLC_DB_PATH): """ raw_db_data = fetch_report_data(db_file_path) processed_db_data = process_db_output(raw_db_data) - - if SITE == "metoffice": - if ESMVAL_CORE_PREVIOUS.exists(): - esmval_core_previous_commit_sha = fetch_git_commits( - ESMVAL_CORE_PREVIOUS - )[0]["sha"] + subheader = create_subheader() + + try: + if site == "dkrz": + # A single commit SHA for each package is expected. + commit_info = get_shas_from_singularity( + ESMVAL_VERSIONS_TODAY, ESMVAL_VERSIONS_YESTERDAY + ) + elif site == "metoffice": + # At least a single commit for each package is expected. There may + # be multiple commits per package, and info for each commit has + # multiple fields. + commit_info = get_commits_from_git(REPOS) + print("commit_info_messages", commit_info) + add_report_message_to_git_commits(commit_info) + print("commit_info_messages", commit_info) else: - esmval_core_previous_commit_sha = None - - if ESMVAL_TOOL_PREVIOUS.exists(): - esmval_tool_previous_commit_sha = fetch_git_commits( - ESMVAL_TOOL_PREVIOUS - )[0]["sha"] - else: - esmval_core_previous_commit_sha = None - - esmval_core_all_commits = fetch_git_commits( - ESMVAL_CORE_CURRENT, esmval_core_previous_commit_sha - ) - esmval_tool_all_commits = fetch_git_commits( - ESMVAL_TOOL_CURRENT, esmval_tool_previous_commit_sha - ) - - subheader = create_subheader() - rendered_html = render_html_report( - subheader=subheader, - report_data=processed_db_data, - esmval_core_commits=esmval_core_all_commits, - esmval_tool_commits=esmval_tool_all_commits, - ) - - elif SITE == "dkrz": - print("Current versons in Python", ESMVAL_VERSIONS_CURRENT) - - else: - subheader = create_subheader() - rendered_html = render_html_report( - subheader=subheader, - report_data=processed_db_data, - ) - + # No commit information for either package. + commit_info = None + # Catch as likely indicate a minor issue e.g. unexpected data content at + # some point in the pipeline. + except (ValueError, KeyError) as err: + "Report generating without commit data. Error while fetching " + f"commit data: {err}" + commit_info = None + + rendered_html = render_html_report( + subheader=subheader, + report_data=processed_db_data, + commit_info=commit_info, + ) write_report_to_file(rendered_html) @@ -195,70 +187,6 @@ def process_db_output(report_data): return sorted_processed_db_data -def add_report_message_to_git_commits(git_commits_info): - """ - Add report messages to a git commit information dictionary. - - Parameters - ---------- - list[dict] - A list of git commits. - """ - git_commits_info[0]["report_flag"] = "Version tested this cycle >>>" - if len(git_commits_info) > 1: - git_commits_info[-1]["report_flag"] = "Version tested last cycle >>>" - - -def fetch_git_commits(package_path, sha=None): - """ - Fetch git commit information for an installed package. - - Parameters - ---------- - package_path : str - Path to a package's git repo. - sha: str | None - Optional. The sha of a previously tested commit. If provided, commits - from HEAD back to the passed sha (inclusive) will be retrieved. - - Returns - ------- - list[dict] - A list of dicts where each dict represents one commit. If ``sha`` is - passed, multiple commits/dicts may be returned. - """ - command = [ - "git", - "log", - "-1", - "--date=iso-strict", - "--pretty=%cd^_^%h^_^%an^_^%s", - ] - - if sha: - command[2] = f"{sha}^..HEAD" - - raw_commit_info = subprocess.run( - command, cwd=package_path, capture_output=True, check=True, text=True - ) - - processed_commit_info = [] - raw_commits = raw_commit_info.stdout.splitlines() - for commit in raw_commits: - split_fields = commit.split("^_^") - processed_commit_info.append( - { - "report_flag": "", - "date": split_fields[0], - "sha": split_fields[1], - "author": split_fields[2], - "message": split_fields[3], - } - ) - add_report_message_to_git_commits(processed_commit_info) - return processed_commit_info - - def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): """ Create the subheader for the HTML report. @@ -279,11 +207,24 @@ def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): return subheader +def add_report_message_to_git_commits(git_commits_info): + """ + Add report messages to a git commit information dictionary. + + Parameters + ---------- + list[dict] + A list of git commits. + """ + git_commits_info[0]["report_flag"] = "Version tested this cycle >>>" + if len(git_commits_info) > 1: + git_commits_info[-1]["report_flag"] = "Version tested last cycle >>>" + + def render_html_report( report_data, subheader, - esmval_core_commits, - esmval_tool_commits, + commit_info, ): """ Render the HTML report using Jinja2. @@ -294,10 +235,7 @@ def render_html_report( The report data to be rendered in the HTML template. subheader : str The subheader for the HTML report. - esmval_core_commits : dict - The ESMValCore commits information. - esmval_tool_commits : dict - The ESMValTool commits information. + package_info : dict Returns ------- @@ -313,8 +251,8 @@ def render_html_report( rendered_html = template.render( subheader=subheader, report_data=report_data, - esmval_core_commits=esmval_core_commits, - esmval_tool_commits=esmval_tool_commits, + esmval_core_commits=commit_info["ESMValCore"]["commits"], + esmval_tool_commits=commit_info["ESMValTool"]["commits"], ) return rendered_html diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/sha_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/sha_via_singularity.py new file mode 100644 index 0000000000..5257e0ccff --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/sha_via_singularity.py @@ -0,0 +1,97 @@ +""" +Functions to fetch git SHA information from singularity containers. +""" + + +def get_shas_from_singularity(dev_version_today, dev_version_yesterday): + """ + Get git SHAs from ``setuptool-scm`` version strings. + + SCM version strings are generated by the command ``esmvaltool version`` and + imported as environment variables. Strings include both packages split by + a newline. E.g. + ``ESMValCore: 2.13.0.dev54+g82d795ec\nESMValTool: 2.13.0.dev66+g53c339c5c`` + where the short SHA-1s are ``82d795ec`` and ``53c339c5c``. + + Parameters + ---------- + dev_version_today : str|None + SCM version string for today's package versions. + dev_version_yesterday : str|None + SCM version string for yesterday's package versions. + + Raises + ------ + ValueError + If SCM versions do not contain consistent SHAs. + + Returns + ------- + tuple[list[dict], list[dict]] + + + """ + shas = { + **extract_scm_shas(dev_version_today, "today"), + **extract_scm_shas(dev_version_yesterday, "yesterday"), + } + if not any(shas.values()): + raise ValueError( + f"No SHAs found: dev_version_today={dev_version_today}" + f"dev_version_yesterday={dev_version_yesterday}" + ) + + elif not (shas["core_today"] or shas["tool_today"]): + raise ValueError( + f"Today's SHAs not found. dev_version_today={dev_version_today}" + ) + + elif not (shas["core_yesterday"] and shas["tool_yesterday"]): + print("Only today's git info is available.") + # commit_info = () # construct tuple + else: + # commit_info = () # construct tuple + pass + # TODO: Complete. Should return commit_info once finalise data structure. + return shas + + +def extract_scm_shas(dev_versions, day): + """ + Extract git SHAs from a SCM version string of combined packages. + + Parameters + ---------- + dev_versions : str | None + The SCM version string for combined packages, split by a newline. + + day: str + The day the version represents. E.g `today` or `yesterday`. + + Notes + ----- + SHA is expected to be 4-40 characters after '+g' and before breaks such + as `.`. For more info on short SHAs: + https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection + + Returns + ------- + sha: dict[str, str | None] + A dictionary where keys are "_" and the values are the + short SHAs, or None if no valid short SHA was found. E.g. + ``{"core_today": abcd123, "tool_today": None}`` + """ + shas = { + f"core_{day}": None, + f"tool_{day}": None, + } + + if isinstance(dev_versions, str): + for line in dev_versions.strip().splitlines(): + sha = line.split("+g")[1].split(".")[0] + if line.startswith("ESMValCore:"): + shas[f"core_{day}"] = sha + elif line.startswith("ESMValTool:"): + shas[f"tool_{day}"] = sha + + return shas diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py index e0358c2093..83cb67eb84 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py @@ -9,7 +9,6 @@ create_subheader, fetch_report_data, process_db_output, - render_html_report, ) @@ -144,21 +143,22 @@ def test_create_subheader(): assert actual == "Cycle start: 2025-01-01 00:01 UTC" -def test_render_html_report(): - mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" - mock_report_data = { - "recipe_1": { - "process_task": {"status": "failed", "style": "color: red"}, - }, - "recipe_2": { - "process_task": {"status": "succeeded", "style": "color: green"}, - "compare_task": {"status": "succeeded", "style": "color: green"}, - }, - "recipe_3": { - "process_task": {"status": "succeeded", "style": "color: green"}, - "compare_task": {"status": "failed", "style": "color: red"}, - }, - } - actual = render_html_report(mock_report_data, mock_subheader) - expected = '\n\n \n Recipe Test Workflow\n \n \n \n \n

Test Results
Recipe Recipe Run
\n

Recipe Test Workflow - Last Run Status

\n

Cycle start: 2025-01-01 00:01 UTC

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
RecipeRecipe RunCompare KGOs
recipe_1failed-
recipe_2succeededsucceeded
recipe_3succeededfailed
\n \n
\n
\n Imprint and\n Privacy Policy\n
\n
\n' - assert actual == expected +# def test_render_html_report(): +# mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" +# mock_report_data = { +# "recipe_1": { +# "process_task": {"status": "failed", "style": "color: red"}, +# }, +# "recipe_2": { +# "process_task": {"status": "succeeded", "style": "color: green"}, +# "compare_task": {"status": "succeeded", "style": "color: green"}, +# }, +# "recipe_3": { +# "process_task": {"status": "succeeded", "style": "color: green"}, +# "compare_task": {"status": "failed", "style": "color: red"}, +# }, +# } +# commit_info = None +# actual = render_html_report(mock_report_data, mock_subheader, commit_info) +# expected = '\n\n \n Recipe Test Workflow\n \n \n \n \n \n

Recipe Test Workflow - Last Run Status

\n

Cycle start: 2025-01-01 00:01 UTC

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
RecipeRecipe RunCompare KGOs
recipe_1failed-
recipe_2succeededsucceeded
recipe_3succeededfailed
\n \n
\n
\n Imprint and\n Privacy Policy\n
\n
\n' +# assert actual == expected diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py new file mode 100644 index 0000000000..0f3814af74 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py @@ -0,0 +1,31 @@ +import pytest + +from bin.sha_via_singularity import extract_scm_shas, get_shas_from_singularity + + +@pytest.fixture() +def mock_scm_version_output(): + return ( + "ESMValCore: 2.13.0.dev54+g82d795ec\n" + "ESMValTool: 2.13.0.dev66+g53c339c5c.d20250523" + ) + + +def test_get_shas_from_singularity(mock_scm_version_output): + dev_version_today = mock_scm_version_output + dev_version_yesterday = mock_scm_version_output + actual = get_shas_from_singularity( + dev_version_today, dev_version_yesterday + ) + expected = { + "core_today": "82d795ec", + "tool_today": "53c339c5c", + "core_yesterday": "82d795ec", + "tool_yesterday": "53c339c5c", + } + assert actual == expected + + +def test_extract_scm_shas_valid(mock_scm_version_output): + actual = extract_scm_shas(mock_scm_version_output, "today") + assert actual == {"core_today": "82d795ec", "tool_today": "53c339c5c"} From 8f3f8b26760a0a8f7d8408b50a85893a7f7256aa Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Fri, 30 May 2025 16:35:19 +0100 Subject: [PATCH 18/40] #4036: WIP commit. Refactored MO/git path working --- .../generate_report/bin/commits_via_git.py | 23 ++++++++- .../bin/generate_html_report.py | 42 +++++++--------- ...singularity.py => shas_via_singularity.py} | 0 .../bin/test_commits_via_git.py | 49 +++++++++++++++++++ ...larity.py => test_shas_via_singularity.py} | 5 +- 5 files changed, 93 insertions(+), 26 deletions(-) rename esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/{sha_via_singularity.py => shas_via_singularity.py} (100%) create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py rename esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/{test_sha_via_singularity.py => test_shas_via_singularity.py} (84%) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py index 32539bd98f..14c5831453 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py @@ -51,6 +51,8 @@ def get_commits_from_git(repos): else: commit_info = get_all_commits_for_today_and_yesterday(repos) + add_report_messages_to_commits(commit_info) + return commit_info @@ -141,7 +143,7 @@ def query_git_log(package_path, sha=None): split_fields = commit.split("^_^") processed_commit_info.append( { - "report_flag": "", + "report_flag": "", # Needed for jinja2 template. "date": split_fields[0], "sha": split_fields[1], "author": split_fields[2], @@ -149,3 +151,22 @@ def query_git_log(package_path, sha=None): } ) return processed_commit_info + + +def add_report_messages_to_commits(commit_info): + """ + Add report messages to a git commit information dictionary. + + Add a report flag + + Parameters + ---------- + commit_info : tuple[list[dict], list[dict] + A tuple containg two list of package commits. + """ + for package_commits in commit_info: + package_commits[0]["report_flag"] = "Version tested this cycle >>>" + if len(package_commits) > 1: + package_commits[-1]["report_flag"] = ( + "Version tested last cycle >>>" + ) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 208bed4c67..d1f3ed1f41 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -6,8 +6,20 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape -from bin.commits_via_git import get_commits_from_git -from bin.sha_via_singularity import get_shas_from_singularity +try: + from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.commits_via_git import ( + get_commits_from_git, + ) +except ImportError: + from commits_via_git import get_commits_from_git + +try: + from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( + get_shas_from_singularity, + ) +except ImportError: + from shas_via_singularity import get_shas_from_singularity + # Load environment variables required at all sites. CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") @@ -23,11 +35,10 @@ elif SITE == "metoffice": REPOS = { "core_today": os.environ.get("ESMVALCORE_DIR"), - "tool_today": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValCore", - "core_yesterday": os.environ.get("ESMVALTOOL_DIR"), + "tool_today": os.environ.get("ESMVALTOOL_DIR"), + "core_yesterday": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValCore", "tool_yesterday": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValTool", } - print("Repos", REPOS) SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" @@ -56,9 +67,6 @@ def main(db_file_path=CYLC_DB_PATH, site=SITE): # be multiple commits per package, and info for each commit has # multiple fields. commit_info = get_commits_from_git(REPOS) - print("commit_info_messages", commit_info) - add_report_message_to_git_commits(commit_info) - print("commit_info_messages", commit_info) else: # No commit information for either package. commit_info = None @@ -207,20 +215,6 @@ def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): return subheader -def add_report_message_to_git_commits(git_commits_info): - """ - Add report messages to a git commit information dictionary. - - Parameters - ---------- - list[dict] - A list of git commits. - """ - git_commits_info[0]["report_flag"] = "Version tested this cycle >>>" - if len(git_commits_info) > 1: - git_commits_info[-1]["report_flag"] = "Version tested last cycle >>>" - - def render_html_report( report_data, subheader, @@ -251,8 +245,8 @@ def render_html_report( rendered_html = template.render( subheader=subheader, report_data=report_data, - esmval_core_commits=commit_info["ESMValCore"]["commits"], - esmval_tool_commits=commit_info["ESMValTool"]["commits"], + esmval_core_commits=commit_info[0], + esmval_tool_commits=commit_info[1], ) return rendered_html diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/sha_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py similarity index 100% rename from esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/sha_via_singularity.py rename to esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py new file mode 100644 index 0000000000..ae14cb86fe --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py @@ -0,0 +1,49 @@ +from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.commits_via_git import ( + add_report_messages_to_commits, +) + + +def test_add_report_message_to_git_commits_today_only(): + mock_commit_info = ( + [{"date": "a", "sha": "abc", "author": "x", "message": "core"}], + [{"date": "a", "sha": "xyz", "author": "y", "message": "tool"}], + ) + add_report_messages_to_commits(mock_commit_info) + expected = "Version tested this cycle >>>" + assert mock_commit_info[0][0]["report_flag"] == expected + assert mock_commit_info[1][0]["report_flag"] == expected + + +def test_add_report_message_to_git_commits_both_days(): + mock_commit_info = ( + [{"date": "a", "sha": "123", "author": "x", "message": "core"}], + [{"date": "a", "sha": "xyz", "author": "y", "message": "tool"}], + ) + add_report_messages_to_commits(mock_commit_info) + expected = "Version tested this cycle >>>" + assert mock_commit_info[0][0]["report_flag"] == expected + assert mock_commit_info[1][0]["report_flag"] == expected + + +def test_add_report_message_git_commits_both_days_multiple_commits(): + mock_commit_info = ( + [ + {"date": "a", "sha": "123", "author": "x", "message": "core_1"}, + {"date": "a", "sha": "456", "author": "z", "message": "core_2"}, + ], + [ + {"date": "b", "sha": "xyz", "author": "y", "message": "tool_1"}, + {"date": "b", "sha": "abc", "author": "z", "message": "tool_2"}, + ], + ) + add_report_messages_to_commits(mock_commit_info) + expected_1 = "Version tested this cycle >>>" + expected_2 = "Version tested last cycle >>>" + assert ( + mock_commit_info[0][0]["report_flag"] + and mock_commit_info[1][0]["report_flag"] + ) == expected_1 + assert ( + mock_commit_info[0][1]["report_flag"] + and mock_commit_info[1][1]["report_flag"] + ) == expected_2 diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py similarity index 84% rename from esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py rename to esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py index 0f3814af74..94f66616ce 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_sha_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py @@ -1,6 +1,9 @@ import pytest -from bin.sha_via_singularity import extract_scm_shas, get_shas_from_singularity +from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( + extract_scm_shas, + get_shas_from_singularity, +) @pytest.fixture() From 9ed4b06f81ee67ecf8917e5b526708ae405d315f Mon Sep 17 00:00:00 2001 From: Chris Billows <96173696+chrisbillows@users.noreply.github.com> Date: Sun, 1 Jun 2025 21:17:09 +0100 Subject: [PATCH 19/40] #4036: Add WIP commits from home machine. Add test main for dkrz (#4079) --- .../generate_report/bin/commits_via_git.py | 18 ++-- .../bin/generate_html_report.py | 90 +++++++++++++------ .../generate_report/bin/report_template.jinja | 43 ++++++--- .../bin/shas_via_singularity.py | 80 ++++++++++------- .../bin/test_generate_html_report.py | 52 ++++++++++- .../bin/test_shas_via_singularity.py | 77 +++++++++++++--- 6 files changed, 264 insertions(+), 96 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py index 14c5831453..797fa21472 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py @@ -4,6 +4,12 @@ import subprocess from pathlib import Path +from typing import NamedTuple + + +class CommitInfo(NamedTuple): + core: list[dict] + tool: list[dict] def get_commits_from_git(repos): @@ -31,19 +37,21 @@ def get_commits_from_git(repos): ``(ESMValCore, ESMValTool)``. Each commit dict has the following fields ``date, sha, author, message``. """ - repo_validity = {key: is_git_repo(path) for key, path in repos.items()} + repo_validity = { + pkg_day: is_git_repo(path) for pkg_day, path in repos.items() + } if not any(repo_validity.values()): raise ValueError("No valid git repos passed") elif not (repo_validity["core_today"] or repo_validity["tool_today"]): - raise ValueError("Today's git commits unavailable.") + raise ValueError("Today's commit info is unavailable.") elif not ( repo_validity["core_yesterday"] and repo_validity["tool_yesterday"] ): - print("Only today's git info is available.") - commit_info = ( + print("Only today's commit info is available.") + commit_info = CommitInfo( query_git_log(repos["core_today"]), query_git_log(repos["tool_today"]), ) @@ -100,7 +108,7 @@ def get_all_commits_for_today_and_yesterday(valid_repos): tool_yesterday_sha = query_git_log(valid_repos["tool_yesterday"][0]["sha"]) tool_commits = query_git_log(valid_repos["tool_today"], tool_yesterday_sha) - return (core_commits, tool_commits) + return CommitInfo(core_commits, tool_commits) def query_git_log(package_path, sha=None): diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index d1f3ed1f41..9c3ff720fb 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -8,6 +8,7 @@ try: from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.commits_via_git import ( + CommitInfo, get_commits_from_git, ) except ImportError: @@ -28,11 +29,16 @@ REPORT_PATH = os.environ.get("REPORT_PATH") SITE = os.environ.get("SITE") +ESMVAL_VERSIONS_TODAY = None +ESMVAL_VERSIONS_YESTERDAY = None +REPOS = None + if SITE == "dkrz": ESMVAL_VERSIONS_TODAY = os.environ.get("ESMVAL_VERSIONS_CURRENT") ESMVAL_VERSIONS_YESTERDAY = os.environ.get("ESMVAL_VERSIONS_PREVIOUS") -elif SITE == "metoffice": + +if SITE == "metoffice": REPOS = { "core_today": os.environ.get("ESMVALCORE_DIR"), "tool_today": os.environ.get("ESMVALTOOL_DIR"), @@ -43,7 +49,15 @@ SQL_QUERY_TASK_STATES = "SELECT name, status FROM task_states" -def main(db_file_path=CYLC_DB_PATH, site=SITE): +def main( + db_file_path=CYLC_DB_PATH, + site=SITE, + report_path=REPORT_PATH, + cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT, + esmval_versions_today=ESMVAL_VERSIONS_TODAY, + esmval_versions_yesterday=ESMVAL_VERSIONS_YESTERDAY, + repos=REPOS, +): """ Main function to generate the HTML report. @@ -51,38 +65,52 @@ def main(db_file_path=CYLC_DB_PATH, site=SITE): ---------- db_file_path : str, default CYLC_DB_FILE_PATH The path to the SQLite database file. + site : str + + report_path : + + cylc_task_cycle_point : + + esmval_versions_today : + + esmval_versions_yesterday : + + repos : + """ + commit_info = None + sha_info = None + raw_db_data = fetch_report_data(db_file_path) processed_db_data = process_db_output(raw_db_data) - subheader = create_subheader() + subheader = create_subheader(cylc_task_cycle_point) + # Commits/SHAs will only be included for these sites. The report will run + # at other sites without commit/SHA information. try: if site == "dkrz": - # A single commit SHA for each package is expected. - commit_info = get_shas_from_singularity( - ESMVAL_VERSIONS_TODAY, ESMVAL_VERSIONS_YESTERDAY + sha_info = get_shas_from_singularity( + esmval_versions_today, esmval_versions_yesterday ) elif site == "metoffice": - # At least a single commit for each package is expected. There may - # be multiple commits per package, and info for each commit has - # multiple fields. - commit_info = get_commits_from_git(REPOS) - else: - # No commit information for either package. - commit_info = None - # Catch as likely indicate a minor issue e.g. unexpected data content at - # some point in the pipeline. - except (ValueError, KeyError) as err: - "Report generating without commit data. Error while fetching " - f"commit data: {err}" - commit_info = None + commit_info = get_commits_from_git(repos) + # Catch the following errors as they are either propagated with specified + # errors or otherwise likely to indicate a minor issue e.g. unexpected + # data content at some point in the pipeline. The report should + # still be output with just the recipe test results. + except (ValueError, KeyError, IndexError) as err: + print( + "Report generating without commit data. Error while fetching commit data: " + f"{err}" + ) rendered_html = render_html_report( subheader=subheader, report_data=processed_db_data, commit_info=commit_info, + commit_shas=sha_info, ) - write_report_to_file(rendered_html) + write_report_to_file(rendered_html, report_path) def fetch_report_data(db_file_path): @@ -195,7 +223,7 @@ def process_db_output(report_data): return sorted_processed_db_data -def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): +def create_subheader(cylc_task_cycle_point): """ Create the subheader for the HTML report. @@ -210,15 +238,16 @@ def create_subheader(cylc_task_cycle_point=CYLC_TASK_CYCLE_POINT): The formatted subheader string. """ parsed_datetime = datetime.strptime(cylc_task_cycle_point, "%Y%m%dT%H%MZ") - formated_datetime = parsed_datetime.strftime("%Y-%m-%d %H:%M") - subheader = f"Cycle start: {formated_datetime} UTC" + formatted_datetime = parsed_datetime.strftime("%Y-%m-%d %H:%M") + subheader = f"Cycle start: {formatted_datetime} UTC" return subheader def render_html_report( report_data, subheader, - commit_info, + commit_info=None, + commit_shas=None, ): """ Render the HTML report using Jinja2. @@ -229,13 +258,17 @@ def render_html_report( The report data to be rendered in the HTML template. subheader : str The subheader for the HTML report. - package_info : dict + commit_info : CommitInfo | None + + commit_shas : dict | None + Returns ------- str The rendered HTML content. """ + commit_info = commit_info or CommitInfo([], []) script_dir = os.path.dirname(os.path.abspath(__file__)) env = Environment( loader=FileSystemLoader(script_dir), @@ -245,13 +278,14 @@ def render_html_report( rendered_html = template.render( subheader=subheader, report_data=report_data, - esmval_core_commits=commit_info[0], - esmval_tool_commits=commit_info[1], + esmval_core_commits=commit_info.core, + esmval_tool_commits=commit_info.tool, + commit_shas=commit_shas, ) return rendered_html -def write_report_to_file(rendered_html, output_file_path=REPORT_PATH): +def write_report_to_file(rendered_html, output_file_path): """ Write the report data to an HTML file. diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index fb43456da1..56c6e6ff35 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -33,6 +33,33 @@

Recipe Test Workflow - Last Run Status

{{ subheader }}

+ {% if commit_shas %} + + + + + + {% if shas['ESMValCore'].get('yesterday') and shas['ESMValTool'].get('yesterday') %} + + {% endif %} + + {% for package, days in shas.items() %} + + + + {% if days.get('yesterday') %} + + {% endif %} + + {% endfor %} +

Commit SHAs

Tested TodayTested Yesterday
{{ package }} + {{ days['today'] }} + + {{ days['yesterday'] }} +
+ {% endif %} + + {% if esmval_core_commits %} @@ -97,20 +124,8 @@ {% for recipe, tasks in report_data.items() %} - - + + {% endfor %}

ESMValCore Commits

{{ recipe }}{{ - tasks["process_task"]["status"] if "process_task" in tasks - else "-" - }}{{ - tasks["compare_task"]["status"] if "compare_task" in tasks - else "-" - }}{{ tasks.get('process_task', {}).get('status', '-') }}{{ tasks.get('compare_task', {}).get('status', '-') }}
diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py index 5257e0ccff..06e3c074e7 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py @@ -2,6 +2,8 @@ Functions to fetch git SHA information from singularity containers. """ +from collections import defaultdict + def get_shas_from_singularity(dev_version_today, dev_version_yesterday): """ @@ -15,45 +17,60 @@ def get_shas_from_singularity(dev_version_today, dev_version_yesterday): Parameters ---------- - dev_version_today : str|None + dev_version_today : str | None SCM version string for today's package versions. - dev_version_yesterday : str|None + dev_version_yesterday : str | None SCM version string for yesterday's package versions. - Raises - ------ - ValueError - If SCM versions do not contain consistent SHAs. - Returns ------- - tuple[list[dict], list[dict]] + sha: dict[str, dict[str, str]] + A dictionary where keys are the package and the values are a dict of days and + short SHAs. E.g. ``{"ESMValCore": {'today': abcd123}...}``. + """ + shas = defaultdict(dict) + for package, day_shas in extract_scm_shas( + dev_version_today, "today" + ).items(): + shas[package].update(day_shas) + for package, day_shas in extract_scm_shas( + dev_version_yesterday, "yesterday" + ).items(): + shas[package].update(day_shas) + validate_shas(shas, dev_version_today, dev_version_yesterday) + return shas +def validate_shas(shas, dev_version_today, dev_version_yesterday): """ - shas = { - **extract_scm_shas(dev_version_today, "today"), - **extract_scm_shas(dev_version_yesterday, "yesterday"), - } - if not any(shas.values()): + Validate extracted SHA combinations. + + These error checks should not be considered exhaustive. + + Raises + ------ + ValueError + If SCM versions do not contain consistent SHAs. Errors can be propagated up + to generate the HTML report without unreliable SHA data. + """ + if not shas["ESMValCore"] and not shas["ESMValTool"]: raise ValueError( f"No SHAs found: dev_version_today={dev_version_today}" f"dev_version_yesterday={dev_version_yesterday}" ) - elif not (shas["core_today"] or shas["tool_today"]): + elif not shas["ESMValCore"].get("today") or not shas["ESMValTool"].get( + "today" + ): raise ValueError( f"Today's SHAs not found. dev_version_today={dev_version_today}" ) - elif not (shas["core_yesterday"] and shas["tool_yesterday"]): - print("Only today's git info is available.") - # commit_info = () # construct tuple - else: - # commit_info = () # construct tuple - pass - # TODO: Complete. Should return commit_info once finalise data structure. - return shas + elif not ( + shas["ESMValCore"].get("yesterday") + and shas["ESMValTool"].get("yesterday") + ): + print("Only today's SHAs are available.") def extract_scm_shas(dev_versions, day): @@ -64,9 +81,8 @@ def extract_scm_shas(dev_versions, day): ---------- dev_versions : str | None The SCM version string for combined packages, split by a newline. - day: str - The day the version represents. E.g `today` or `yesterday`. + The day the versions were tested on. E.g ``today`` or ``yesterday``. Notes ----- @@ -76,22 +92,22 @@ def extract_scm_shas(dev_versions, day): Returns ------- - sha: dict[str, str | None] - A dictionary where keys are "_" and the values are the - short SHAs, or None if no valid short SHA was found. E.g. - ``{"core_today": abcd123, "tool_today": None}`` + sha: dict[str, dict[str, str]] | dict[str, dict[None]] + A dictionary where keys are the package and the values are a dict of days and + short SHAs. E.g. ``{"ESMValCore": {'today': abcd123}...}`` or None if no SHAs + were found. """ shas = { - f"core_{day}": None, - f"tool_{day}": None, + "ESMValCore": {}, + "ESMValTool": {}, } if isinstance(dev_versions, str): for line in dev_versions.strip().splitlines(): sha = line.split("+g")[1].split(".")[0] if line.startswith("ESMValCore:"): - shas[f"core_{day}"] = sha + shas["ESMValCore"][day] = sha elif line.startswith("ESMValTool:"): - shas[f"tool_{day}"] = sha + shas["ESMValTool"][day] = sha return shas diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py index c8c314b491..0c4958319c 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py @@ -1,6 +1,8 @@ +import os import sqlite3 import tempfile from pathlib import Path +from unittest.mock import patch import pytest @@ -8,8 +10,12 @@ SQL_QUERY_TASK_STATES, create_subheader, fetch_report_data, + main, process_db_output, ) +from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.test_shas_via_singularity import ( + mock_scm_version_output, +) @pytest.fixture() @@ -44,6 +50,27 @@ def test_mock_cylc_db_and_sql_query(mock_cylc_db): assert actual.fetchall() == expected +def test_main_for_site_dkrz(mock_cylc_db): + # Duplicate input doesn't impact test. + esmval_versions_today = mock_scm_version_output() + esmval_version_yesterday = mock_scm_version_output() + with patch.dict(os.environ, {"MY_ENV_VAR": "mocked_value"}): + with tempfile.TemporaryDirectory() as temp_directory: + actual = main( + mock_cylc_db, + "dkrz", + temp_directory, + "20250601T2015Z", + esmval_versions_today, + esmval_version_yesterday, + None, + ) + assert actual is None + expected_report = Path(temp_directory) / "status_report.html" + assert expected_report.exists() + # report_content = expected_report.read_text() + + def test_fetch_report_data(mock_cylc_db): actual = fetch_report_data(mock_cylc_db) expected = [ @@ -143,7 +170,7 @@ def test_create_subheader(): assert actual == "Cycle start: 2025-01-01 00:01 UTC" -# def test_render_html_report(): +# def test_render_html_report_no_commits_no_shas(): # mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" # mock_report_data = { # "recipe_1": { @@ -158,7 +185,26 @@ def test_create_subheader(): # "compare_task": {"status": "failed", "style": "color: red"}, # }, # } -# commit_info = None -# actual = render_html_report(mock_report_data, mock_subheader, commit_info) +# actual = render_html_report(mock_report_data, mock_subheader) # expected = '\n\n \n Recipe Test Workflow\n \n \n \n \n \n

Recipe Test Workflow - Last Run Status

\n

Cycle start: 2025-01-01 00:01 UTC

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
RecipeRecipe RunCompare KGOs
recipe_1failed-
recipe_2succeededsucceeded
recipe_3succeededfailed
\n \n
\n
\n Imprint and\n Privacy Policy\n
\n
\n' # assert actual == expected + + +# def test_render_html_report_partial(): +# mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" +# mock_report_data = { +# "recipe_1": { +# "process_task": {"status": "failed", "style": "color: red"}, +# }, +# "recipe_2": { +# "process_task": {"status": "succeeded", "style": "color: green"}, +# "compare_task": {"status": "succeeded", "style": "color: green"}, +# }, +# "recipe_3": { +# "process_task": {"status": "succeeded", "style": "color: green"}, +# "compare_task": {"status": "failed", "style": "color: red"}, +# }, +# } +# commit_info = None +# actual = render_html_report(mock_report_data, mock_subheader, commit_info) +# assert expected in actual diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py index 94f66616ce..30b9239f3e 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py @@ -3,32 +3,81 @@ from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( extract_scm_shas, get_shas_from_singularity, + validate_shas, ) -@pytest.fixture() def mock_scm_version_output(): + """ + A valid mock SCM version string. + + A function is used to allow safe in-test mutation and to allow the result + to be passed as parametrized test values. + + Returns + ------- + str + A valid mock SCM version string. + """ return ( "ESMValCore: 2.13.0.dev54+g82d795ec\n" "ESMValTool: 2.13.0.dev66+g53c339c5c.d20250523" ) -def test_get_shas_from_singularity(mock_scm_version_output): - dev_version_today = mock_scm_version_output - dev_version_yesterday = mock_scm_version_output +@pytest.mark.parametrize( + "mock_day_version_today, mock_day_version_yesterday, expected", + [ + ( + mock_scm_version_output(), + None, + { + "ESMValCore": {"today": "82d795ec"}, + "ESMValTool": {"today": "53c339c5c"}, + }, + ), + ( + mock_scm_version_output(), + mock_scm_version_output(), + { + "ESMValCore": {"today": "82d795ec", "yesterday": "82d795ec"}, + "ESMValTool": {"today": "53c339c5c", "yesterday": "53c339c5c"}, + }, + ), + ], +) +def test_get_shas_from_singularity_valid_shas( + mock_day_version_today, mock_day_version_yesterday, expected +): actual = get_shas_from_singularity( - dev_version_today, dev_version_yesterday + mock_day_version_today, mock_day_version_yesterday ) - expected = { - "core_today": "82d795ec", - "tool_today": "53c339c5c", - "core_yesterday": "82d795ec", - "tool_yesterday": "53c339c5c", - } assert actual == expected -def test_extract_scm_shas_valid(mock_scm_version_output): - actual = extract_scm_shas(mock_scm_version_output, "today") - assert actual == {"core_today": "82d795ec", "tool_today": "53c339c5c"} +@pytest.mark.parametrize( + "shas, expected_message", + [ + ( + {"ESMValCore": {}, "ESMValTool": {}}, + "No SHAs found: dev_version_today=", + ), + ( + {"ESMValCore": {"today": "sha"}, "ESMValTool": {}}, + "Today's SHAs not found. dev_version_today=", + ), + ], +) +def test_get_shas_from_singularity_invalid_shas(shas, expected_message): + with pytest.raises(ValueError, match=expected_message): + # The unprocessed scm version strings are passed to the function purely for + # error logging. Here None is used. + validate_shas(shas, None, None) + + +def test_extract_scm_shas_valid(): + actual = extract_scm_shas(mock_scm_version_output(), "today") + assert actual == { + "ESMValCore": {"today": "82d795ec"}, + "ESMValTool": {"today": "53c339c5c"}, + } From 15ebd45c5fdd279d9407674c2041c73aa22fc87b Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Mon, 2 Jun 2025 11:25:22 +0100 Subject: [PATCH 20/40] #4036: DKRZ site test passing --- .../bin/generate_html_report.py | 13 +- .../generate_report/bin/report_template.jinja | 8 +- .../bin/test_generate_html_report.py | 134 ++++++++++-------- 3 files changed, 83 insertions(+), 72 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index b2a527f828..cc59f1270a 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -106,7 +106,7 @@ def main( subheader=subheader, report_data=processed_db_data, commit_info=commit_info, - commit_shas=sha_info, + sha_info=sha_info, ) write_report_to_file(rendered_html, report_path) @@ -248,12 +248,7 @@ def create_subheader(cylc_task_cycle_point): return subheader -def render_html_report( - report_data, - subheader, - commit_info=None, - commit_shas=None, -): +def render_html_report(report_data, subheader, commit_info, sha_info): """ Render the HTML report using Jinja2. @@ -265,7 +260,7 @@ def render_html_report( The subheader for the HTML report. commit_info : CommitInfo | None - commit_shas : dict | None + sha_info : dict | None Returns @@ -285,7 +280,7 @@ def render_html_report( report_data=report_data, esmval_core_commits=commit_info.core, esmval_tool_commits=commit_info.tool, - commit_shas=commit_shas, + sha_info=sha_info, ) return rendered_html diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index 56c6e6ff35..577b986956 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -33,17 +33,17 @@

Recipe Test Workflow - Last Run Status

{{ subheader }}

- {% if commit_shas %} + {% if sha_info %} - {% if shas['ESMValCore'].get('yesterday') and shas['ESMValTool'].get('yesterday') %} + {% if sha_info['ESMValCore'].get('yesterday') and sha_info['ESMValTool'].get('yesterday') %} {% endif %} - {% for package, days in shas.items() %} + {% for package, days in sha_info.items() %} {% endfor %}

Commit SHAs
Tested TodayTested Yesterday
{{ package }} @@ -110,9 +110,9 @@
+ {% endif %}
- {% endif %} diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py index f96c8b8c34..ff6853b1c9 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py @@ -1,9 +1,8 @@ -import os import sqlite3 import tempfile +from collections import namedtuple from contextlib import contextmanager from pathlib import Path -from unittest.mock import patch import pytest @@ -12,11 +11,26 @@ fetch_report_data, main, process_db_output, + render_html_report, ) from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.test_shas_via_singularity import ( mock_scm_version_output, ) +MockDbData = namedtuple("MockDbdata", ["cycle", "row_data"]) + + +@pytest.fixture +def mock_db_data_single_cycle(): + cycle = "20250521T0100Z" + row_data = [ + ("process_recipe_1", "succeeded", cycle), + ("compare_recipe_1", "succeeded", cycle), + ("process_recipe_2", "succeeded", cycle), + ("compare_recipe_2", "failed", cycle), + ] + return MockDbData(cycle, row_data) + @contextmanager def mock_db_with_passed_values(row_data): @@ -48,43 +62,43 @@ def mock_db_with_passed_values(row_data): yield path_to_synthetic_db -def test_main_for_site_dkrz(mock_cylc_db): - # Duplicate input doesn't impact test. +def test_main_for_site_dkrz(mock_db_data_single_cycle): esmval_versions_today = mock_scm_version_output() - esmval_version_yesterday = mock_scm_version_output() - with patch.dict(os.environ, {"MY_ENV_VAR": "mocked_value"}): + # Duplicate input doesn't impact test. + esmval_versions_yesterday = mock_scm_version_output() + + with mock_db_with_passed_values( + mock_db_data_single_cycle.row_data + ) as mock_cylc_db: with tempfile.TemporaryDirectory() as temp_directory: + report_path = Path(temp_directory) / "status_report.html" actual = main( mock_cylc_db, "dkrz", - temp_directory, - "20250601T2015Z", + report_path, + mock_db_data_single_cycle.cycle, esmval_versions_today, - esmval_version_yesterday, + esmval_versions_yesterday, None, ) assert actual is None - expected_report = Path(temp_directory) / "status_report.html" - assert expected_report.exists() + assert report_path.exists() # report_content = expected_report.read_text() -def test_fetch_report_data_single_cycle(): - mock_cycle = "20250521T0100Z" - mock_data = [ - ("process_recipe_1", "succeeded", mock_cycle), - ("compare_recipe_1", "succeeded", mock_cycle), - ("process_recipe_2", "succeeded", mock_cycle), - ("compare_recipe_2", "failed", mock_cycle), - ] +def test_fetch_report_data_single_cycle(mock_db_data_single_cycle): expected = [ ("process_recipe_1", "succeeded"), ("compare_recipe_1", "succeeded"), ("process_recipe_2", "succeeded"), ("compare_recipe_2", "failed"), ] - with mock_db_with_passed_values(mock_data) as mock_cylc_db: - actual = fetch_report_data(mock_cylc_db, mock_cycle) + with mock_db_with_passed_values( + mock_db_data_single_cycle.row_data + ) as mock_cylc_db: + actual = fetch_report_data( + mock_cylc_db, mock_db_data_single_cycle.cycle + ) assert actual == expected @@ -199,41 +213,43 @@ def test_create_subheader(): assert actual == "Cycle start: 2025-01-01 00:01 UTC" -# def test_render_html_report_no_commits_no_shas(): -# mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" -# mock_report_data = { -# "recipe_1": { -# "process_task": {"status": "failed", "style": "color: red"}, -# }, -# "recipe_2": { -# "process_task": {"status": "succeeded", "style": "color: green"}, -# "compare_task": {"status": "succeeded", "style": "color: green"}, -# }, -# "recipe_3": { -# "process_task": {"status": "succeeded", "style": "color: green"}, -# "compare_task": {"status": "failed", "style": "color: red"}, -# }, -# } -# actual = render_html_report(mock_report_data, mock_subheader) -# expected = '\n\n \n Recipe Test Workflow\n \n \n \n \n

Test Results

\n

Recipe Test Workflow - Last Run Status

\n

Cycle start: 2025-01-01 00:01 UTC

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
RecipeRecipe RunCompare KGOs
recipe_1failed-
recipe_2succeededsucceeded
recipe_3succeededfailed
\n \n
\n
\n Imprint and\n Privacy Policy\n
\n
\n' -# assert actual == expected - - -# def test_render_html_report_partial(): -# mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" -# mock_report_data = { -# "recipe_1": { -# "process_task": {"status": "failed", "style": "color: red"}, -# }, -# "recipe_2": { -# "process_task": {"status": "succeeded", "style": "color: green"}, -# "compare_task": {"status": "succeeded", "style": "color: green"}, -# }, -# "recipe_3": { -# "process_task": {"status": "succeeded", "style": "color: green"}, -# "compare_task": {"status": "failed", "style": "color: red"}, -# }, -# } -# commit_info = None -# actual = render_html_report(mock_report_data, mock_subheader, commit_info) -# assert expected in actual +@pytest.mark.skip(reason="Finish reimplementation") +def test_render_html_report_no_commits_no_shas(): + mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" + mock_report_data = { + "recipe_1": { + "process_task": {"status": "failed", "style": "color: red"}, + }, + "recipe_2": { + "process_task": {"status": "succeeded", "style": "color: green"}, + "compare_task": {"status": "succeeded", "style": "color: green"}, + }, + "recipe_3": { + "process_task": {"status": "succeeded", "style": "color: green"}, + "compare_task": {"status": "failed", "style": "color: red"}, + }, + } + actual = render_html_report(mock_report_data, mock_subheader) + expected = '\n\n \n Recipe Test Workflow\n \n \n \n \n \n

Recipe Test Workflow - Last Run Status

\n

Cycle start: 2025-01-01 00:01 UTC

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
RecipeRecipe RunCompare KGOs
recipe_1failed-
recipe_2succeededsucceeded
recipe_3succeededfailed
\n \n
\n
\n Imprint and\n Privacy Policy\n
\n
\n' + assert actual == expected + + +@pytest.mark.skip(reason="Finish reimplementation") +def test_render_html_report_partial(): + mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" + mock_report_data = { + "recipe_1": { + "process_task": {"status": "failed", "style": "color: red"}, + }, + "recipe_2": { + "process_task": {"status": "succeeded", "style": "color: green"}, + "compare_task": {"status": "succeeded", "style": "color: green"}, + }, + "recipe_3": { + "process_task": {"status": "succeeded", "style": "color: green"}, + "compare_task": {"status": "failed", "style": "color: red"}, + }, + } + commit_info = None + actual = render_html_report(mock_report_data, mock_subheader, commit_info) + assert "" in actual From 52f83808d47e6c53a38c42b99fa77c3d69678313 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Mon, 9 Jun 2025 10:58:44 +0100 Subject: [PATCH 21/40] #4036: dkrz tweaks. run every 10mins for testing --- .../app/generate_report/opt/rose-app-dkrz.conf | 2 ++ esmvaltool/utils/recipe_test_workflow/flow.cylc | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf index f229d01ac3..43d73abe5e 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf @@ -2,6 +2,8 @@ default=set -euo pipefail =export ESMVAL_VERSIONS_CURRENT=$(CAPTURE_OUTPUT=True env-file esmvaltool version) =echo $ESMVAL_VERSIONS_CURRENT + =export ESMVAL_VERSIONS_PREVIOUS=$(QUIET_MODE=True CAPTURE_OUTPUT=True CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" env-file esmvaltool version) + =echo $ESMVAL_VERSIONS_PREVIOUS =env-file generate_html_report.py =if [ "${PRODUCTION}" = "True" ]; then rsync -av "${REPORT_PATH}" "${VM_PATH}"; echo "HTML report copied from ${REPORT_PATH} to ${VM_PATH}"; fi diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index 8da114b1a5..98c174f465 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -36,7 +36,7 @@ process:fail? | compare:finish => generate_report process:fail? | compare:finish => generate_report """ - T01 = """ + PT10M = """ @wall_clock => get_esmval => configure configure => process? => compare? configure => process? => compare? @@ -51,7 +51,7 @@ [runtime] [[root]] script = rose task-run - env-script = "eval $(rose task-env --cycle-offset=P1D)" + env-script = "eval $(rose task-env --cycle-offset=PT10M)" [[[environment]]] ENV_NAME = {{ ENV_NAME }} USER_CONFIG_DIR = ${ROSE_DATAC}/config_dir From 5a6f33914126ba537b42018488562c18f8167061 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Mon, 9 Jun 2025 14:26:28 +0100 Subject: [PATCH 22/40] 4036: Minor DKRZ fixes (running). Create container versions bash script --- .../bin/generate_html_report.py | 10 ++++++---- .../bin/set_container_versions.sh | 19 +++++++++++++++++++ .../generate_report/opt/rose-app-dkrz.conf | 5 +---- 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100755 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index cc59f1270a..2450f21be9 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os import sqlite3 +import traceback from datetime import datetime from pathlib import Path @@ -12,7 +13,7 @@ get_commits_from_git, ) except ImportError: - from commits_via_git import get_commits_from_git + from commits_via_git import CommitInfo, get_commits_from_git try: from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( @@ -96,11 +97,12 @@ def main( # errors or otherwise likely to indicate a minor issue e.g. unexpected # data content at some point in the pipeline. The report should # still be output with just the recipe test results. - except (ValueError, KeyError, IndexError) as err: + except (ValueError, KeyError, IndexError): print( - "Report generating without commit data. Error while fetching commit data: " - f"{err}" + "Report generating without commit data. Error while fetching " + "commit data. See std.err log for details." ) + traceback.print_exc() rendered_html = render_html_report( subheader=subheader, diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh new file mode 100755 index 0000000000..cffd4dc502 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Send the output from 'set -x' to 'stdout' rather than 'stderr'. +BASH_XTRACEFD=1 +set -eux + +echo Running new bash script + +export CAPTURE_OUTPUT=True +export QUIET_MODE=True +export YESTERDAYS_CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" + +export ESMVAL_VERSIONS_CURRENT=$(env-file esmvaltool version) + +if [[ -f ${YESTERDAYS_CONTAINER_PATH} ]]; then + export ESMVAL_VERSIONS_PREVIOUS=$(CONTAINER_PATH=${YESTERDAYS_CONTAINER_PATH} env-file esmvaltool version) +fi + +echo ESMVal Current Versions: $ESMVAL_VERSIONS_CURRENT +echo ESMVal Previous Versions: $ESMVAL_VERSIONS_PREVIOUS diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf index 43d73abe5e..6764471eca 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf @@ -1,9 +1,6 @@ [command] default=set -euo pipefail - =export ESMVAL_VERSIONS_CURRENT=$(CAPTURE_OUTPUT=True env-file esmvaltool version) - =echo $ESMVAL_VERSIONS_CURRENT - =export ESMVAL_VERSIONS_PREVIOUS=$(QUIET_MODE=True CAPTURE_OUTPUT=True CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" env-file esmvaltool version) - =echo $ESMVAL_VERSIONS_PREVIOUS + =set_container_versions.sh =env-file generate_html_report.py =if [ "${PRODUCTION}" = "True" ]; then rsync -av "${REPORT_PATH}" "${VM_PATH}"; echo "HTML report copied from ${REPORT_PATH} to ${VM_PATH}"; fi From 469d32b06942e9b41ecccb6e5903e5f9bfb4d512 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Mon, 9 Jun 2025 14:45:20 +0100 Subject: [PATCH 23/40] #4036: Fix shellcheck errors --- .../app/generate_report/bin/set_container_versions.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh index cffd4dc502..39aa77d555 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh @@ -9,11 +9,13 @@ export CAPTURE_OUTPUT=True export QUIET_MODE=True export YESTERDAYS_CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" -export ESMVAL_VERSIONS_CURRENT=$(env-file esmvaltool version) +ESMVAL_VERSIONS_CURRENT=$(env-file esmvaltool version) +export ESMVAL_VERSIONS_CURRENT if [[ -f ${YESTERDAYS_CONTAINER_PATH} ]]; then - export ESMVAL_VERSIONS_PREVIOUS=$(CONTAINER_PATH=${YESTERDAYS_CONTAINER_PATH} env-file esmvaltool version) + ESMVAL_VERSIONS_PREVIOUS=$(CONTAINER_PATH=${YESTERDAYS_CONTAINER_PATH} env-file esmvaltool version) + export ESMVAL_VERSIONS_PREVIOUS fi -echo ESMVal Current Versions: $ESMVAL_VERSIONS_CURRENT -echo ESMVal Previous Versions: $ESMVAL_VERSIONS_PREVIOUS +echo ESMVal Current Versions: "$ESMVAL_VERSIONS_CURRENT" +echo ESMVal Previous Versions: "$ESMVAL_VERSIONS_PREVIOUS" From c5bdcdae1497d52e2967dcb946e4c435903b11fb Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Tue, 10 Jun 2025 12:21:08 +0200 Subject: [PATCH 24/40] #4036: Fix dkrz bash script --- .../generate_report/bin/set_container_versions.sh | 15 ++++----------- .../app/generate_report/opt/rose-app-dkrz.conf | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh index 39aa77d555..3cc49cf7db 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh @@ -1,21 +1,14 @@ #!/bin/bash # Send the output from 'set -x' to 'stdout' rather than 'stderr'. BASH_XTRACEFD=1 -set -eux +set -eu -echo Running new bash script +YESTERDAYS_CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" -export CAPTURE_OUTPUT=True -export QUIET_MODE=True -export YESTERDAYS_CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" - -ESMVAL_VERSIONS_CURRENT=$(env-file esmvaltool version) +ESMVAL_VERSIONS_CURRENT=$(QUIET_MODE=True CAPTURE_OUTPUT=True env-file esmvaltool version) export ESMVAL_VERSIONS_CURRENT if [[ -f ${YESTERDAYS_CONTAINER_PATH} ]]; then - ESMVAL_VERSIONS_PREVIOUS=$(CONTAINER_PATH=${YESTERDAYS_CONTAINER_PATH} env-file esmvaltool version) + ESMVAL_VERSIONS_PREVIOUS=$(QUIET_MODE=True CAPTURE_OUTPUT=True CONTAINER_PATH=${YESTERDAYS_CONTAINER_PATH} env-file esmvaltool version) export ESMVAL_VERSIONS_PREVIOUS fi - -echo ESMVal Current Versions: "$ESMVAL_VERSIONS_CURRENT" -echo ESMVal Previous Versions: "$ESMVAL_VERSIONS_PREVIOUS" diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf index 6764471eca..e70a4d0273 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/opt/rose-app-dkrz.conf @@ -1,6 +1,6 @@ [command] default=set -euo pipefail - =set_container_versions.sh + =source set_container_versions.sh =env-file generate_html_report.py =if [ "${PRODUCTION}" = "True" ]; then rsync -av "${REPORT_PATH}" "${VM_PATH}"; echo "HTML report copied from ${REPORT_PATH} to ${VM_PATH}"; fi From 87eb896192cbabfe7b13a7c51f7a340bd14169ff Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Tue, 10 Jun 2025 14:08:55 +0200 Subject: [PATCH 25/40] #4036: Unify imports --- .../app/generate_report/bin/generate_html_report.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 2450f21be9..e3cf8a4438 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -12,14 +12,11 @@ CommitInfo, get_commits_from_git, ) -except ImportError: - from commits_via_git import CommitInfo, get_commits_from_git - -try: from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( get_shas_from_singularity, ) except ImportError: + from commits_via_git import CommitInfo, get_commits_from_git from shas_via_singularity import get_shas_from_singularity From f06ce76ec247dd4b1a93ffe0348774a67249c29b Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Tue, 10 Jun 2025 16:17:21 +0100 Subject: [PATCH 26/40] #4036: Refactor sha extraction code --- .../bin/shas_via_singularity.py | 144 ++++++++---------- .../bin/test_shas_via_singularity.py | 30 ++-- 2 files changed, 75 insertions(+), 99 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py index 06e3c074e7..81f6004ea3 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py @@ -2,18 +2,18 @@ Functions to fetch git SHA information from singularity containers. """ -from collections import defaultdict +import re -def get_shas_from_singularity(dev_version_today, dev_version_yesterday): +def get_shas_from_singularity(dev_versions_today, dev_versions_yesterday): """ Get git SHAs from ``setuptool-scm`` version strings. SCM version strings are generated by the command ``esmvaltool version`` and imported as environment variables. Strings include both packages split by - a newline. E.g. - ``ESMValCore: 2.13.0.dev54+g82d795ec\nESMValTool: 2.13.0.dev66+g53c339c5c`` - where the short SHA-1s are ``82d795ec`` and ``53c339c5c``. + a newline. E.g. ``ESMValCore: 2.13.0.dev54+g82d795ec\nESMValTool: + 2.13.0.dev66+g53c339c5c\n`` where the short SHA-1s are ``82d795ec`` and + ``53c339c5c``. Parameters ---------- @@ -22,92 +22,74 @@ def get_shas_from_singularity(dev_version_today, dev_version_yesterday): dev_version_yesterday : str | None SCM version string for yesterday's package versions. + Notes + ----- + The regex used expects short SHAs between "+g" and a new line in the SCM + version strings. A branch with uncommited changes would break this pattern + but that should not be possible from a container. + + More on setuptools scm's versioning scheme here under "Default verisioning + scheme: + https://setuptools-scm.readthedocs.io/en/latest/usage/ + + More on short SHA-1s in general here: + https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection + Returns ------- - sha: dict[str, dict[str, str]] - A dictionary where keys are the package and the values are a dict of days and - short SHAs. E.g. ``{"ESMValCore": {'today': abcd123}...}``. + all_shas: dict[str, dict[str, str]] + A dictionary where keys are the package and the values are a dict of + days and short SHAs. E.g. + ``{"ESMValCore": {"today": "abcd123", "yesterday": "efgh456"}...}``. """ - shas = defaultdict(dict) - for package, day_shas in extract_scm_shas( - dev_version_today, "today" - ).items(): - shas[package].update(day_shas) - for package, day_shas in extract_scm_shas( - dev_version_yesterday, "yesterday" - ).items(): - shas[package].update(day_shas) - validate_shas(shas, dev_version_today, dev_version_yesterday) - return shas - - -def validate_shas(shas, dev_version_today, dev_version_yesterday): - """ - Validate extracted SHA combinations. + all_shas = {"ESMValCore": {}, "ESMValTool": {}} + pkg_versions_by_day = [("today", dev_versions_today)] - These error checks should not be considered exhaustive. - - Raises - ------ - ValueError - If SCM versions do not contain consistent SHAs. Errors can be propagated up - to generate the HTML report without unreliable SHA data. - """ - if not shas["ESMValCore"] and not shas["ESMValTool"]: - raise ValueError( - f"No SHAs found: dev_version_today={dev_version_today}" - f"dev_version_yesterday={dev_version_yesterday}" - ) - - elif not shas["ESMValCore"].get("today") or not shas["ESMValTool"].get( - "today" - ): - raise ValueError( - f"Today's SHAs not found. dev_version_today={dev_version_today}" - ) - - elif not ( - shas["ESMValCore"].get("yesterday") - and shas["ESMValTool"].get("yesterday") - ): + if dev_versions_yesterday: + pkg_versions_by_day.append(("yesterday", dev_versions_yesterday)) + else: print("Only today's SHAs are available.") + for day, package_versions in pkg_versions_by_day: + shas = re.findall(r"\+g(.*?)$", package_versions, re.MULTILINE) + if len(shas) == 2: + all_shas["ESMValCore"][day] = shas[0] + all_shas["ESMValTool"][day] = shas[1] + else: + print(f"Unexpected SHA format for {day}: {package_versions}") + + validate_all_shas(all_shas, dev_versions_today, dev_versions_yesterday) + return all_shas -def extract_scm_shas(dev_versions, day): + +def validate_all_shas(all_shas, dev_version_today, dev_version_yesterday): """ - Extract git SHAs from a SCM version string of combined packages. + Validate extracted SHA combinations. + + These error checks should not be considered exhaustive. Parameters ---------- - dev_versions : str | None - The SCM version string for combined packages, split by a newline. - day: str - The day the versions were tested on. E.g ``today`` or ``yesterday``. - - Notes - ----- - SHA is expected to be 4-40 characters after '+g' and before breaks such - as `.`. For more info on short SHAs: - https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection + all_shas : dict[str, dict[str, str]] + A dictionary where keys are the package and the values are a dict of + days and short SHAs. + dev_version_today : str | None + SCM version string for today's package versions. Required for logging. + dev_version_yesterday : str | None + SCM version string for yesterday's package versions. Required for + logging. - Returns - ------- - sha: dict[str, dict[str, str]] | dict[str, dict[None]] - A dictionary where keys are the package and the values are a dict of days and - short SHAs. E.g. ``{"ESMValCore": {'today': abcd123}...}`` or None if no SHAs - were found. + Raises + ------ + ValueError + If SCM versions do not contain consistent SHAs. Errors can be + propagated up to generate the HTML report without unreliable SHA data. """ - shas = { - "ESMValCore": {}, - "ESMValTool": {}, - } - - if isinstance(dev_versions, str): - for line in dev_versions.strip().splitlines(): - sha = line.split("+g")[1].split(".")[0] - if line.startswith("ESMValCore:"): - shas["ESMValCore"][day] = sha - elif line.startswith("ESMValTool:"): - shas["ESMValTool"][day] = sha - - return shas + partial_message = ( + f"dev_version_today={dev_version_today} dev_version_yesterday=" + f"{dev_version_yesterday} pkg_shas={all_shas}" + ) + packages = ["ESMValCore", "ESMValTool"] + + if not all(all_shas[package] for package in packages): + raise ValueError("Missing SHAs: " + partial_message) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py index 30b9239f3e..d91ebd8cc7 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py @@ -1,9 +1,8 @@ import pytest from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( - extract_scm_shas, get_shas_from_singularity, - validate_shas, + validate_all_shas, ) @@ -21,7 +20,7 @@ def mock_scm_version_output(): """ return ( "ESMValCore: 2.13.0.dev54+g82d795ec\n" - "ESMValTool: 2.13.0.dev66+g53c339c5c.d20250523" + "ESMValTool: 2.13.0.dev66+g53c339c5c\n" ) @@ -46,12 +45,15 @@ def mock_scm_version_output(): ), ], ) -def test_get_shas_from_singularity_valid_shas( +def test_get_shas_from_singularity_and_validate_for_valid_shas( mock_day_version_today, mock_day_version_yesterday, expected ): actual = get_shas_from_singularity( mock_day_version_today, mock_day_version_yesterday ) + # The unprocessed scm version strings are passed to the function purely + # for error logging. Here 'None' is used. + validate_all_shas(actual, None, None) assert actual == expected @@ -60,24 +62,16 @@ def test_get_shas_from_singularity_valid_shas( [ ( {"ESMValCore": {}, "ESMValTool": {}}, - "No SHAs found: dev_version_today=", + "Missing SHAs: dev_version_today=", ), ( {"ESMValCore": {"today": "sha"}, "ESMValTool": {}}, - "Today's SHAs not found. dev_version_today=", + "Missing SHAs: dev_version_today=", ), ], ) -def test_get_shas_from_singularity_invalid_shas(shas, expected_message): +def test_validate_all_shas_for_invalid_shas(shas, expected_message): with pytest.raises(ValueError, match=expected_message): - # The unprocessed scm version strings are passed to the function purely for - # error logging. Here None is used. - validate_shas(shas, None, None) - - -def test_extract_scm_shas_valid(): - actual = extract_scm_shas(mock_scm_version_output(), "today") - assert actual == { - "ESMValCore": {"today": "82d795ec"}, - "ESMValTool": {"today": "53c339c5c"}, - } + # The unprocessed scm version strings are passed to the function purely + # for error logging. Here 'None' is used. + validate_all_shas(shas, None, None) From 625f0ad0ce23500cf92ffa9b29312fbf77555978 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 11 Jun 2025 14:23:02 +0100 Subject: [PATCH 27/40] #4036: Various minor refactors, minor reformatting etc. --- .../generate_report/bin/commits_via_git.py | 93 +++++++++++-------- .../bin/generate_html_report.py | 66 +++++++------ .../generate_report/bin/report_template.jinja | 4 - .../bin/set_container_versions.sh | 2 +- .../bin/shas_via_singularity.py | 4 +- .../bin/test_shas_via_singularity.py | 2 +- .../app/generate_report/rose-app.conf | 3 +- .../utils/recipe_test_workflow/flow.cylc | 8 +- 8 files changed, 96 insertions(+), 86 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py index 797fa21472..3eb308a7f9 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py @@ -1,13 +1,23 @@ -""" -Functions to fetch commit information from local git repositories. -""" +"Functions to fetch commit information from local git repositories." import subprocess +from dataclasses import dataclass from pathlib import Path -from typing import NamedTuple -class CommitInfo(NamedTuple): +@dataclass +class CommitInfo: + """ + A dataclass to hold commit information for ESMValCore and ESMValTool. + + Attributes + ---------- + core : list[dict] + A list of dictionaries containing commit information for ESMValCore. + tool : list[dict] + A list of dictionaries containing commit information for ESMValTool. + """ + core: list[dict] tool: list[dict] @@ -19,36 +29,36 @@ def get_commits_from_git(repos): Parameters ---------- repos : dict[str, str | None] - A dictionary where keys are in the form ``repo_day`` and values are - the path to the repo, or None. E.g. ``{"core_today": "path/to/repo", - "core_yesterday": None}`` + A dictionary where keys are in the form ``_`` and values + are the path to the repo, or None. E.g. + ``{"core_today": "path/to/repo", "core_yesterday": None}`` Raises ------ ValueError - If repos does not contain enough valid git directories to fetch - useable commit data. + If repos does not contain any valid git directories, or does not + contain valid git directories for today. Returns ------- - tuple[list[dict], list[dict]] - A tuple of two lists of dictionaries. Each dictionary include - information on a commit or a range of commits for - ``(ESMValCore, ESMValTool)``. Each commit dict has the following fields - ``date, sha, author, message``. + CommitInfo + A CommitInfo dataclass containing two lists of dictionaries, one for + each of ESMValCore and ESMValTool. Each commit dict has the following + fields ``date, sha, author, message``. """ repo_validity = { pkg_day: is_git_repo(path) for pkg_day, path in repos.items() } if not any(repo_validity.values()): - raise ValueError("No valid git repos passed") + raise ValueError("No valid git repos found.") - elif not (repo_validity["core_today"] or repo_validity["tool_today"]): + if not (repo_validity["core_today"] or repo_validity["tool_today"]): raise ValueError("Today's commit info is unavailable.") - elif not ( - repo_validity["core_yesterday"] and repo_validity["tool_yesterday"] + if not ( + repo_validity.get("core_yesterday") + or repo_validity.get("tool_yesterday") ): print("Only today's commit info is available.") commit_info = CommitInfo( @@ -70,7 +80,7 @@ def is_git_repo(path): Parameters ---------- - path : str|None + path : str | None Path to a git repo, or None. Returns @@ -86,9 +96,10 @@ def is_git_repo(path): def get_all_commits_for_today_and_yesterday(valid_repos): """ - Fetch information on a range of commits. + Fetch ``git log`` information for a range of commits. - Fetches a range of commits + Expects a valid git repo for both ESMValCore and ESMValTool for today + and yesterday. Parameters ---------- @@ -97,18 +108,20 @@ def get_all_commits_for_today_and_yesterday(valid_repos): Returns ------- - tuple[list[dict], list[dict]] - A tuple with two lists of dictionaries that include a range of commit - information for ``(ESMValCore, ESMValTool)``. - + CommitInfo + A CommitInfo dataclass containing two lists of dictionaries, one for + each of ESMValCore and ESMValTool. """ - core_yesterday_sha = query_git_log(valid_repos["core_yesterday"][0]["sha"]) - core_commits = query_git_log(valid_repos["core_today"], core_yesterday_sha) - - tool_yesterday_sha = query_git_log(valid_repos["tool_yesterday"][0]["sha"]) - tool_commits = query_git_log(valid_repos["tool_today"], tool_yesterday_sha) - - return CommitInfo(core_commits, tool_commits) + commit_info = CommitInfo([], []) + for package in ["core", "tool"]: + yesterdays_sha = query_git_log(valid_repos[f"{package}_yesterday"])[0][ + "sha" + ] + commit_range = query_git_log( + valid_repos[f"{package}_today"], yesterdays_sha + ) + setattr(commit_info, package, commit_range) + return commit_info def query_git_log(package_path, sha=None): @@ -128,8 +141,8 @@ def query_git_log(package_path, sha=None): list[dict] A list of dicts where each dict represents one commit. Each commit has the following fields ``date, sha, author, message``. If ``sha`` is - not passed, one commit is returned. If ``sha`` is passed, a minimum of - two commits is returned. + not passed, one commit is returned. If ``sha`` is passed, multiple + commits may be returned. """ command = [ "git", @@ -163,16 +176,14 @@ def query_git_log(package_path, sha=None): def add_report_messages_to_commits(commit_info): """ - Add report messages to a git commit information dictionary. - - Add a report flag + Add report messages to a CommitInfo dataclass. Parameters ---------- - commit_info : tuple[list[dict], list[dict] - A tuple containg two list of package commits. + commit_info : CommitInfo + A CommitInfo dataclass containing two lists of package commits. """ - for package_commits in commit_info: + for package_commits in [commit_info.core, commit_info.tool]: package_commits[0]["report_flag"] = "Version tested this cycle >>>" if len(package_commits) > 1: package_commits[-1]["report_flag"] = ( diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index e3cf8a4438..16d490651c 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +"Generate a HTML summary report from a Cylc SQLite database." + import os import sqlite3 import traceback @@ -7,6 +9,7 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape +# Import from the ESMValTool package for testing. try: from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.commits_via_git import ( CommitInfo, @@ -15,6 +18,7 @@ from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( get_shas_from_singularity, ) +# Import locally for running in Cylc. except ImportError: from commits_via_git import CommitInfo, get_commits_from_git from shas_via_singularity import get_shas_from_singularity @@ -23,7 +27,7 @@ # Load environment variables required at all sites. CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") -CYLC_TASK_CYCLE_YESTERDAY = os.environ.get("ROSE_DATACP1D") +CYLC_TASK_CYCLE_YESTERDAY = os.environ.get("CYLC_TASK_CYCLE_YESTERDAY") REPORT_PATH = os.environ.get("REPORT_PATH") SITE = os.environ.get("SITE") @@ -40,9 +44,11 @@ REPOS = { "core_today": os.environ.get("ESMVALCORE_DIR"), "tool_today": os.environ.get("ESMVALTOOL_DIR"), - "core_yesterday": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValCore", - "tool_yesterday": Path(CYLC_TASK_CYCLE_YESTERDAY) / "ESMValTool", } + if CYLC_TASK_CYCLE_YESTERDAY: + path_to_yesterdays_cycle = Path(CYLC_TASK_CYCLE_YESTERDAY) + REPOS["core_yesterday"] = path_to_yesterdays_cycle / "ESMValCore" + REPOS["tool_yesterday"] = path_to_yesterdays_cycle / "ESMValTool" def main( @@ -62,22 +68,24 @@ def main( db_file_path : str, default CYLC_DB_FILE_PATH The path to the SQLite database file. site : str - - report_path : - - cylc_task_cycle_point : - - esmval_versions_today : - - esmval_versions_yesterday : - - repos : - + The site the Recipe Test Workflow is being run at. + report_path : str + The path to output the HTML report. + cylc_task_cycle_point : str + The cycle point of the task as a string in ISO8601 format. + esmval_versions_today : str | None + The path to today's singularity container, if the site uses a + singularity container, or None. + esmval_versions_yesterday : str | None + The path to yesterday's singularity container, if the site uses a + singularity container and it exists, or None. + repos : dict[str, str] | None + A dictionary of git repos if the site uses git repos, or None. """ - commit_info = None sha_info = None + commit_info = None - raw_db_data = fetch_report_data(db_file_path) + raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) processed_db_data = process_db_output(raw_db_data) subheader = create_subheader(cylc_task_cycle_point) @@ -90,14 +98,13 @@ def main( ) elif site == "metoffice": commit_info = get_commits_from_git(repos) - # Catch the following errors as they are either propagated with specified - # errors or otherwise likely to indicate a minor issue e.g. unexpected - # data content at some point in the pipeline. The report should - # still be output with just the recipe test results. + # Catch the following errors so the report generates without commit/SHA + # information. These errors are either propagated on purpose or + # indicate a probable minor issue. except (ValueError, KeyError, IndexError): print( - "Report generating without commit data. Error while fetching " - "commit data. See std.err log for details." + "Report generating with results only. Error while fetching commit " + "data. See std.err log for details." ) traceback.print_exc() @@ -110,7 +117,7 @@ def main( write_report_to_file(rendered_html, report_path) -def fetch_report_data(db_file_path, target_cycle_point=CYLC_TASK_CYCLE_POINT): +def fetch_report_data(db_file_path, target_cycle_point): """ Fetch report data for a single cycle from the Cylc SQLite database. @@ -118,9 +125,8 @@ def fetch_report_data(db_file_path, target_cycle_point=CYLC_TASK_CYCLE_POINT): ---------- db_file_path : str The path to the SQLite database file. - target_cycle_point : str, default CYLC_TASK_CYCLE_POINT - The cycle point to collect data for. Defaults to the current cylc - cycle. + target_cycle_point : str + The cycle point to collect data for. Returns ------- @@ -258,9 +264,11 @@ def render_html_report(report_data, subheader, commit_info, sha_info): subheader : str The subheader for the HTML report. commit_info : CommitInfo | None - + The commit information for ESMValCore and ESMValTool, if the site uses + git repos, or None. sha_info : dict | None - + The SHA information for ESMValCore and ESMValTool, if the site uses + singularity containers, or None. Returns ------- @@ -292,7 +300,7 @@ def write_report_to_file(rendered_html, output_file_path): ---------- rendered_html : str The rendered HTML content. - output_file_path : str, default OUTPUT_FILE_PATH + output_file_path : str The path to the output HTML file. """ with open(output_file_path, "w") as file: diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index 577b986956..ee804e53bf 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -28,7 +28,6 @@ color: white; } -

Recipe Test Workflow - Last Run Status

{{ subheader }}

@@ -59,7 +58,6 @@ {% endif %} - {% if esmval_core_commits %} @@ -111,9 +109,7 @@ {% endfor %}

ESMValCore Commits

{% endif %} -
- diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh index 3cc49cf7db..718af57735 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/set_container_versions.sh @@ -3,7 +3,7 @@ BASH_XTRACEFD=1 set -eu -YESTERDAYS_CONTAINER_PATH="${ROSE_DATACPT10M}/container/esmvaltool.sif" +YESTERDAYS_CONTAINER_PATH="${CYLC_TASK_CYCLE_YESTERDAY}/container/esmvaltool.sif" ESMVAL_VERSIONS_CURRENT=$(QUIET_MODE=True CAPTURE_OUTPUT=True env-file esmvaltool version) export ESMVAL_VERSIONS_CURRENT diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py index 81f6004ea3..3c96ebe17d 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py @@ -1,6 +1,4 @@ -""" -Functions to fetch git SHA information from singularity containers. -""" +"Functions to fetch git SHA information from singularity containers." import re diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py index d91ebd8cc7..bdfed78ef0 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py @@ -8,7 +8,7 @@ def mock_scm_version_output(): """ - A valid mock SCM version string. + Valid mock SCM version string. A function is used to allow safe in-test mutation and to allow the result to be passed as parametrized test values. diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf index b62e4505bb..cf3db61d85 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/rose-app.conf @@ -1,3 +1,2 @@ [command] -default=export ESMVAL_VERSIONS=$(QUIET_MODE=True env-file esmvaltool version) - =env-file generate_html_report.py +default=env-file generate_html_report.py diff --git a/esmvaltool/utils/recipe_test_workflow/flow.cylc b/esmvaltool/utils/recipe_test_workflow/flow.cylc index 81bc2ade32..e44a3b9344 100644 --- a/esmvaltool/utils/recipe_test_workflow/flow.cylc +++ b/esmvaltool/utils/recipe_test_workflow/flow.cylc @@ -156,16 +156,14 @@ [[generate_report]] [[[environment]]] + SITE = {{ SITE }} # By default opt configurations must exist. Paretheses make the opt # file optional, which is required as only a DKRZ opt file exists. - SITE = {{ SITE }} ROSE_APP_OPT_CONF_KEYS = ({{ SITE }}) CYLC_DB_PATH = ${CYLC_WORKFLOW_RUN_DIR}/log/db - PRODUCTION = {{ PRODUCTION }} REPORT_PATH = ${ROSE_DATAC}/status_report.html - SHARE_BIN = ${CYLC_WORKFLOW_SHARE_DIR}/bin - ENV_FILE = ${SHARE_BIN}/env-file - SING_ENV_FILE = ${SHARE_BIN}/singularity-env-file + CYLC_TASK_CYCLE_YESTERDAY = ${ROSE_DATACPT10M} + PRODUCTION = {{ PRODUCTION }} [[housekeeping]] platform = localhost From baaa803343bd7dff94ff584c067704435171b38c Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 11 Jun 2025 16:30:23 +0100 Subject: [PATCH 28/40] #4036: Improve jinja template layout --- .../generate_report/bin/report_template.jinja | 135 ++++++++++-------- 1 file changed, 73 insertions(+), 62 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index ee804e53bf..b274c933e7 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -3,63 +3,75 @@ Recipe Test Workflow - - + .table-full-width { + width: 100%; + } + + .table-half-width { + width: 50%; + } + +

Recipe Test Workflow - Last Run Status

{{ subheader }}

{% if sha_info %} -

Test Results
- +

Commit SHAs

+ - {% if sha_info['ESMValCore'].get('yesterday') and sha_info['ESMValTool'].get('yesterday') %} - - {% endif %} + {%- if sha_info['ESMValCore'].get('yesterday') and sha_info['ESMValTool'].get('yesterday') -%} + + {%- endif -%} - {% for package, days in sha_info.items() %} + {%- for package, days in sha_info.items() -%} - {% if days.get('yesterday') %} + {%- if days.get('yesterday') -%} - {% endif %} + {%- endif -%} - {% endfor %} + {%- endfor -%}

Commit SHAs
Tested TodayTested YesterdayTested Yesterday
{{ package }} {{ days['today'] }} {{ days['yesterday'] }}
{% endif %} {% if esmval_core_commits %} - +
@@ -68,24 +80,26 @@ - {% for commit in esmval_core_commits %} + {%- for commit in esmval_core_commits -%} - {% if commit['report_flag'] %} + {%- if commit['report_flag'] -%} - {% else %} + {%- else -%} - {% endif %} + {%- endif -%} - + - {% endfor %} + {%- endfor -%}

ESMValCore Commits
Author Commit message
{{ commit['report_flag'] }}{{ commit['date'] }}{{ commit['sha'] }} + {{ commit['sha'] }} + {{ commit['author'] }} {{ commit['message']}}
{% endif %} {% if esmval_tool_commits %} - +
@@ -94,48 +108,45 @@ - {% for commit in esmval_tool_commits %} - - {% if commit['report_flag'] %} - - {% else %} - - {% endif %} - - - - - - {% endfor %} + {%- for commit in esmval_tool_commits -%} + + {%- if commit['report_flag'] -%} + + {%- else -%} + + {%- endif -%} + + + + + + {%- endfor -%}

ESMValTool Commits
Author Commit message
{{ commit['report_flag'] }}{{ commit['date'] }}{{ commit['sha'] }}{{ commit['author'] }} {{ commit['message']}}
{{ commit['report_flag'] }}{{ commit['date'] }}{{ commit['sha'] }}{{ commit['author'] }} {{ commit['message']}}
{% endif %} +
- + +
- {% for recipe, tasks in report_data.items() %} + {%- for recipe, tasks in report_data.items() -%} - {% endfor %} + {%- endfor -%}

Test Results
Recipe Recipe Run Compare KGOs
{{ recipe }} {{ tasks.get('process_task', {}).get('status', '-') }} {{ tasks.get('compare_task', {}).get('status', '-') }}
From 6cf94cf6e845641bbce1fe5ec3a86518aee5bc3f Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 11 Jun 2025 16:40:06 +0100 Subject: [PATCH 29/40] #4036: Codacy tweaks --- .../app/generate_report/bin/commits_via_git.py | 2 +- .../app/generate_report/bin/generate_html_report.py | 2 +- .../app/generate_report/bin/shas_via_singularity.py | 2 +- .../app/generate_report/bin/test_shas_via_singularity.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py index 3eb308a7f9..c2bf077829 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py @@ -1,4 +1,4 @@ -"Functions to fetch commit information from local git repositories." +"""Functions to fetch commit information from local git repositories.""" import subprocess from dataclasses import dataclass diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 16d490651c..fe6ef7d589 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"Generate a HTML summary report from a Cylc SQLite database." +"""Generate a HTML summary report from a Cylc SQLite database.""" import os import sqlite3 diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py index 3c96ebe17d..85c975a6ec 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py @@ -1,4 +1,4 @@ -"Functions to fetch git SHA information from singularity containers." +"""Functions to fetch git SHA information from singularity containers.""" import re diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py index bdfed78ef0..79b2b79fcf 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py @@ -8,7 +8,7 @@ def mock_scm_version_output(): """ - Valid mock SCM version string. + Return a valid mock SCM version string. A function is used to allow safe in-test mutation and to allow the result to be passed as parametrized test values. From 26c3cceac3ce914cb650f715c1684a9bef8dc600 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Tue, 17 Jun 2025 16:37:59 +0100 Subject: [PATCH 30/40] #4036: WIP commit adding GitHub API calls --- .../generate_report/bin/commits_via_git.py | 191 -------------- .../generate_report/bin/fetch_commit_info.py | 191 ++++++++++++++ .../bin/generate_html_report.py | 102 ++++++-- .../app/generate_report/bin/shas_via_git.py | 70 ++++++ .../bin/test_commits_via_git.py | 2 +- .../bin/test_fetch_commit_info.py | 238 ++++++++++++++++++ setup.cfg | 6 - 7 files changed, 580 insertions(+), 220 deletions(-) delete mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_git.py create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py delete mode 100644 setup.cfg diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py deleted file mode 100644 index c2bf077829..0000000000 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/commits_via_git.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Functions to fetch commit information from local git repositories.""" - -import subprocess -from dataclasses import dataclass -from pathlib import Path - - -@dataclass -class CommitInfo: - """ - A dataclass to hold commit information for ESMValCore and ESMValTool. - - Attributes - ---------- - core : list[dict] - A list of dictionaries containing commit information for ESMValCore. - tool : list[dict] - A list of dictionaries containing commit information for ESMValTool. - """ - - core: list[dict] - tool: list[dict] - - -def get_commits_from_git(repos): - """ - Fetch commit information from local git repos. - - Parameters - ---------- - repos : dict[str, str | None] - A dictionary where keys are in the form ``_`` and values - are the path to the repo, or None. E.g. - ``{"core_today": "path/to/repo", "core_yesterday": None}`` - - Raises - ------ - ValueError - If repos does not contain any valid git directories, or does not - contain valid git directories for today. - - Returns - ------- - CommitInfo - A CommitInfo dataclass containing two lists of dictionaries, one for - each of ESMValCore and ESMValTool. Each commit dict has the following - fields ``date, sha, author, message``. - """ - repo_validity = { - pkg_day: is_git_repo(path) for pkg_day, path in repos.items() - } - - if not any(repo_validity.values()): - raise ValueError("No valid git repos found.") - - if not (repo_validity["core_today"] or repo_validity["tool_today"]): - raise ValueError("Today's commit info is unavailable.") - - if not ( - repo_validity.get("core_yesterday") - or repo_validity.get("tool_yesterday") - ): - print("Only today's commit info is available.") - commit_info = CommitInfo( - query_git_log(repos["core_today"]), - query_git_log(repos["tool_today"]), - ) - - else: - commit_info = get_all_commits_for_today_and_yesterday(repos) - - add_report_messages_to_commits(commit_info) - - return commit_info - - -def is_git_repo(path): - """ - Check a passed value is a valid git directory. - - Parameters - ---------- - path : str | None - Path to a git repo, or None. - - Returns - ------- - bool - If the passed value is a valid git directory. - """ - try: - return Path(path).expanduser().resolve().joinpath(".git").is_dir() - except (TypeError, OSError): - return False - - -def get_all_commits_for_today_and_yesterday(valid_repos): - """ - Fetch ``git log`` information for a range of commits. - - Expects a valid git repo for both ESMValCore and ESMValTool for today - and yesterday. - - Parameters - ---------- - valid_repos : dict[str, str] - A dict of valid git repos. - - Returns - ------- - CommitInfo - A CommitInfo dataclass containing two lists of dictionaries, one for - each of ESMValCore and ESMValTool. - """ - commit_info = CommitInfo([], []) - for package in ["core", "tool"]: - yesterdays_sha = query_git_log(valid_repos[f"{package}_yesterday"])[0][ - "sha" - ] - commit_range = query_git_log( - valid_repos[f"{package}_today"], yesterdays_sha - ) - setattr(commit_info, package, commit_range) - return commit_info - - -def query_git_log(package_path, sha=None): - """ - Use ``git log`` to fetch commit information from a local git repo. - - Parameters - ---------- - package_path : str - Path to a valid git repo. - sha: str | None - Optional. The sha of a previously tested commit. If provided, commits - from HEAD back to the passed sha (inclusive) will be retrieved. - - Returns - ------- - list[dict] - A list of dicts where each dict represents one commit. Each commit has - the following fields ``date, sha, author, message``. If ``sha`` is - not passed, one commit is returned. If ``sha`` is passed, multiple - commits may be returned. - """ - command = [ - "git", - "log", - "-1", - "--date=iso-strict", - "--pretty=%cd^_^%h^_^%an^_^%s", - ] - - if sha: - command[2] = f"{sha}^..HEAD" - - raw_commit_info = subprocess.run( - command, cwd=package_path, capture_output=True, check=True, text=True - ) - processed_commit_info = [] - raw_commits = raw_commit_info.stdout.splitlines() - for commit in raw_commits: - split_fields = commit.split("^_^") - processed_commit_info.append( - { - "report_flag": "", # Needed for jinja2 template. - "date": split_fields[0], - "sha": split_fields[1], - "author": split_fields[2], - "message": split_fields[3], - } - ) - return processed_commit_info - - -def add_report_messages_to_commits(commit_info): - """ - Add report messages to a CommitInfo dataclass. - - Parameters - ---------- - commit_info : CommitInfo - A CommitInfo dataclass containing two lists of package commits. - """ - for package_commits in [commit_info.core, commit_info.tool]: - package_commits[0]["report_flag"] = "Version tested this cycle >>>" - if len(package_commits) > 1: - package_commits[-1]["report_flag"] = ( - "Version tested last cycle >>>" - ) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py new file mode 100644 index 0000000000..3a574b5211 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py @@ -0,0 +1,191 @@ +""" +Fetches commit details from the GitHub API. +""" + +import os + +import requests + +tested_today = "662e792984d6577ae52e6931e794386ce508960c" + +GITHUB_API_URL = "https://api.github.com" +GITHUB_API_PERSONAL_ACCESS_TOKEN = os.environ.get( + "GITHUB_API_PERSONAL_ACCESS_TOKEN" +) +HEADERS = { + "authorization": f"token {GITHUB_API_PERSONAL_ACCESS_TOKEN}", + # Suggested here: + # https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#get-a-commit--parameters + # Explanation here: + # https://docs.github.com/en/rest/overview/resources-in-the-rest-api#http-HEADERS + "accept": "application/vnd.github+json", +} + + +def fetch_commit_details_from_github_api( + shas_by_package_and_day, headers=HEADERS +): + """ + Fetch commit details from the GitHub API for the given SHAs. + + Parameters + ---------- + shas_by_package_and_day : dict[str, dict[str, str]] + A dictionary where keys are the package names and values are dictionaries + with days as keys and SHAs as values. E.g. + {"ESMValCore": {"today": "abcd123", "yesterday": "efgh456"}...}. + + Returns + ------- + dict[str, list[dict]] + A dictionary where keys are the package names and values are lists of + commit details for each day. E.g. + {"ESMValCore": [{"sha": "abcd123", ...}, ...], "ESMValTool": [...]} + """ + commit_details_by_package = {} + for package, shas_by_day in shas_by_package_and_day.items(): + if shas_by_day.get("yesterday") is None or shas_by_day.get( + "today" + ) == shas_by_day.get("yesterday"): + raw_commit = fetch_single_commit( + package, "ESMValGroup", headers, shas_by_day["today"] + ) + # commit_info = process_commit_info(raw_commit) + commit_details_by_package[package] = [raw_commit] + else: + raw_commits = fetch_range_of_commits( + package, + "ESMValGroup", + headers, + newer_sha=shas_by_day["today"], + older_sha=shas_by_day["yesterday"], + ) + commit_details_by_package[package] = raw_commits + # commit_info = process_commit_info(commit_info) + return commit_details_by_package + + +def fetch_single_commit(repo, owner, headers, sha): + """ + Fetch details of a single commit from the GitHub API. + + Parameters: + ---------- + repo: str + The name of the repository. E.g. "ESMValTool" + owner: str + The owner of the repository. E.g. "ESMValGroup" + headers: dict + Headers to include in the request. + sha: str + The SHA of the commit to fetch details for. + + Raises + ------ + HTTPError + If the commit is not found or if the request fails etc. + + Returns + ------- + dict + The raw commit data if found, otherwise None. + """ + url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/commits/{sha}" + response = requests.get(url, headers=headers) + response.raise_for_status() # Raise a HTTP error for bad responses + raw_commit = response.json() + return raw_commit + + +def fetch_range_of_commits(repo, owner, headers, newer_sha, older_sha): + """ + Fetch details for a range of commits from the GitHub API. + + The endpoint will return a range of commits in chronlogical order, from + the newer SHA to the older SHA. The function fetches batches of 10 commits + to avoid hitting the API rate limits. NOTE: The GitHub API will raise a + HTTPError if the newer SHA is not found. + + Raises + ------ + HTTPError + If the newer SHA is not found (or if the request fails etc.) + ValueError + If too many pages are fetched, indicating a potential infinite loop. + + Parameters: + ---------- + repo : str + The name of the repository. E.g. "ESMValTool" + owner : str + The owner of the repository. E.g. "ESMValGroup" + headers : dict + Headers to include in the request. + newer_sha : str + The SHA of the first commit to start fetching details for. + older_sha : str + The SHA of the commit to stop fetching at. + + """ + url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/commits/" + params = { + "per_page": 10, + "sha": newer_sha, + } + range_raw_commits = [] + page = 1 + + fetched_end_sha = False + while not fetched_end_sha: + params["page"] = page + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() # Raise a HTTP error for bad responses + + page_raw_commits = response.json() + + for raw_commit in page_raw_commits: + range_raw_commits.append(raw_commit) + if raw_commit["sha"].startswith(older_sha): + fetched_end_sha = True + break + + page += 1 + if page > 5: + raise ValueError( + "Too many pages fetched, likely an infinite loop. Check the " + "newer and older SHAs." + ) + return range_raw_commits + + +def process_commit_info(raw_commit_info): + """ + Extract required commit details. + + Parameters: + ----------- + raw_commit_info : dict | list[dict] + Raw commit information from the GitHub API. Either a single commit or a + list of commits. + + Returns + ------- + dict + Processed commit information with relevant details. + """ + + if not isinstance(raw_commit_info, list): + raw_commit_info = [raw_commit_info] + + processed_commit_info = [] + for raw_commit in raw_commit_info: + processed_comit = { + "sha": raw_commit["sha"][:7], + "author": raw_commit["commit"]["author"]["name"], + "message": raw_commit["commit"]["message"], + "date": raw_commit["commit"]["author"]["date"], + "url": raw_commit["html_url"], + "author_avatar": raw_commit["author"]["avatar_url"], + } + processed_commit_info.append(processed_comit) + return processed_commit_info diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index fe6ef7d589..1f875de85e 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -8,19 +8,24 @@ from pathlib import Path from jinja2 import Environment, FileSystemLoader, select_autoescape +from requests.exceptions import HTTPError # Import from the ESMValTool package for testing. try: - from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.commits_via_git import ( + from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info import ( + fetch_commit_details_from_github_api, + ) + from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_git import ( CommitInfo, - get_commits_from_git, + get_shas_from_git, ) from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( get_shas_from_singularity, ) # Import locally for running in Cylc. except ImportError: - from commits_via_git import CommitInfo, get_commits_from_git + from fetch_commit_info import fetch_commit_details_from_github_api + from shas_via_git import get_shas_from_git from shas_via_singularity import get_shas_from_singularity @@ -42,13 +47,13 @@ if SITE == "metoffice": REPOS = { - "core_today": os.environ.get("ESMVALCORE_DIR"), - "tool_today": os.environ.get("ESMVALTOOL_DIR"), + "ESMValCore_today": os.environ.get("ESMVALCORE_DIR"), + "ESMValTool_today": os.environ.get("ESMVALTOOL_DIR"), } if CYLC_TASK_CYCLE_YESTERDAY: path_to_yesterdays_cycle = Path(CYLC_TASK_CYCLE_YESTERDAY) - REPOS["core_yesterday"] = path_to_yesterdays_cycle / "ESMValCore" - REPOS["tool_yesterday"] = path_to_yesterdays_cycle / "ESMValTool" + REPOS["ESMValCore_yesterday"] = path_to_yesterdays_cycle / "ESMValCore" + REPOS["ESMValTool_yesterday"] = path_to_yesterdays_cycle / "ESMValTool" def main( @@ -82,13 +87,7 @@ def main( repos : dict[str, str] | None A dictionary of git repos if the site uses git repos, or None. """ - sha_info = None commit_info = None - - raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) - processed_db_data = process_db_output(raw_db_data) - subheader = create_subheader(cylc_task_cycle_point) - # Commits/SHAs will only be included for these sites. The report will run # at other sites without commit/SHA information. try: @@ -97,24 +96,48 @@ def main( esmval_versions_today, esmval_versions_yesterday ) elif site == "metoffice": - commit_info = get_commits_from_git(repos) + sha_info = get_shas_from_git(repos) # Catch the following errors so the report generates without commit/SHA # information. These errors are either propagated on purpose or # indicate a probable minor issue. - except (ValueError, KeyError, IndexError): + except (ValueError, KeyError, IndexError, HTTPError): print( "Report generating with results only. Error while fetching commit " "data. See std.err log for details." ) traceback.print_exc() + if sha_info: + commit_info = fetch_commit_details_from_github_api(sha_info) + print(commit_info) - rendered_html = render_html_report( - subheader=subheader, - report_data=processed_db_data, - commit_info=commit_info, - sha_info=sha_info, - ) - write_report_to_file(rendered_html, report_path) + # raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) + # processed_db_data = process_db_output(raw_db_data) + # subheader = create_subheader(cylc_task_cycle_point) + + # rendered_html = render_html_report( + # subheader=subheader, + # report_data=processed_db_data, + # commit_info=commit_info, + # ) + + # write_report_to_file(rendered_html, report_path) + + +def add_report_messages_to_commits(commit_info): + """ + Add report messages to a CommitInfo dataclass. + + Parameters + ---------- + commit_info : CommitInfo + A CommitInfo dataclass containing two lists of package commits. + """ + for package_commits in [commit_info.core, commit_info.tool]: + package_commits[0]["report_flag"] = "Version tested this cycle >>>" + if len(package_commits) > 1: + package_commits[-1]["report_flag"] = ( + "Version tested last cycle >>>" + ) def fetch_report_data(db_file_path, target_cycle_point): @@ -233,6 +256,41 @@ def process_db_output(report_data): return sorted_processed_db_data +def copy_debug_log_to_vm(failed_task): + """ """ + # TODO: Current work is in python-playground/lumberjack.py + pass + + +def add_debug_log_for_failed_tasks(processed_db_data): + """ + { + "recipe_1": { + "process_task": { + "status": "succeeded", + "style": "color: green", + "debug_log": "path/to/debug.log", + }, + "compare_task": { + "status": "failed", + "style": "color: red", + "debug_log": "path/to/debug.log", + }, + } + }, + """ + + for recipe, tasks in processed_db_data.items(): + for task_name, task_data in tasks.items(): + if task_data["status"] == "failed": + # Assuming the debug log path is constructed from the recipe name + # and task name. Adjust as necessary. + debug_log_path = ( + f"/path/to/debug/logs/{recipe}/{task_name}.log" + ) + task_data["debug_log"] = debug_log_path + + def create_subheader(cylc_task_cycle_point): """ Create the subheader for the HTML report. diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_git.py new file mode 100644 index 0000000000..3cd573cdf7 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_git.py @@ -0,0 +1,70 @@ +"""Functions to fetch commit information from local git repositories.""" + +import subprocess +from pathlib import Path + + +def get_shas_from_git(repos): + """ + Fetch commit information from local git repos. + + Parameters + ---------- + repos : dict[str, str | None] + A dictionary where keys are in the form ``_`` and values + are the path to the repo, or None. E.g. + ``{"core_today": "path/to/repo", "core_yesterday": None}`` + Returns + ------- + dict + A dictionary where keys are the package and the values are a dict of + days and short SHAs. E.g. + ``{"ESMValCore": {"today": "abcd123", "yesterday": "efgh456"}...}``. + """ + all_shas = {"ESMValCore": {}, "ESMValTool": {}} + for package, repo_path in repos.items(): + if repo_path is not None and is_git_repo(repo_path): + package, day = package.split("_") + all_shas[package][day] = fetch_sha_from_git_log(repo_path) + return all_shas + + +def is_git_repo(path): + """ + Check a passed value is a valid git directory. + + Parameters + ---------- + path : str | None + Path to a git repo, or None. + + Returns + ------- + bool + If the passed value is a valid git directory. + """ + try: + return Path(path).expanduser().resolve().joinpath(".git").is_dir() + except (TypeError, OSError): + return False + + +def fetch_sha_from_git_log(package_path): + """ + Use ``git log`` to get the SHA of the latest commit in a local git repo. + + Parameters + ---------- + package_path : str + Path to a valid git repo. + + Returns + ------- + sha: str + The SHA of the latest commit in the repo. + """ + command = ["git", "log", "-1", "--pretty=%H"] + sha = subprocess.run( + command, cwd=package_path, capture_output=True, check=True, text=True + ) + return sha.stdout.strip() diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py index ae14cb86fe..cce556781b 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py @@ -1,4 +1,4 @@ -from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.commits_via_git import ( +from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_git import ( add_report_messages_to_commits, ) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py new file mode 100644 index 0000000000..ad202c79ca --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py @@ -0,0 +1,238 @@ +from unittest.mock import Mock, call, patch + +import pytest + +from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info import ( + fetch_commit_details_from_github_api, + fetch_range_of_commits, + fetch_single_commit, + process_commit_info, +) + + +@pytest.mark.parametrize( + "mock_shas_by_package_and_day", + [ + { + "ESMValCore": {"today": "abcd123"}, + "ESMValTool": {"today": "ijkl789"}, + }, + { + "ESMValCore": {"today": "abcd123", "yesterday": "abcd123"}, + "ESMValTool": {"today": "ijkl789", "yesterday": "ijkl789"}, + }, + ], + ids=["todays_shas_only", "todays_shas_same_as_yesterdays"], +) +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.fetch_single_commit" +) +def test_fetch_commit_details_from_github_api_single_commits( + mock_fetch_single, mock_shas_by_package_and_day +): + mock_fetch_single.return_value = {"mock_response": "data"} + mock_headers = "headers" + + actual = fetch_commit_details_from_github_api( + mock_shas_by_package_and_day, mock_headers + ) + + assert actual == { + "ESMValCore": [{"mock_response": "data"}], + "ESMValTool": [{"mock_response": "data"}], + } + + assert mock_fetch_single.call_count == 2 + assert mock_fetch_single.call_args_list == [ + call("ESMValCore", "ESMValGroup", mock_headers, "abcd123"), + call("ESMValTool", "ESMValGroup", mock_headers, "ijkl789"), + ] + + +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.fetch_range_of_commits" +) +def test_fetch_commit_details_from_github_api_range_of_commits( + mock_fetch_range, +): + mock_fetch_range.return_value = [{"mock_response": "data"}] + mock_headers = "headers" + mock_shas_by_package_and_day = { + "ESMValCore": {"today": "abcd123", "yesterday": "efgh456"}, + "ESMValTool": {"today": "ijkl789", "yesterday": "mnop012"}, + } + + actual = fetch_commit_details_from_github_api( + mock_shas_by_package_and_day, mock_headers + ) + + assert actual == { + "ESMValCore": [{"mock_response": "data"}], + "ESMValTool": [{"mock_response": "data"}], + } + + assert mock_fetch_range.call_count == 2 + assert mock_fetch_range.call_args_list == [ + call( + "ESMValCore", + "ESMValGroup", + mock_headers, + newer_sha="abcd123", + older_sha="efgh456", + ), + call( + "ESMValTool", + "ESMValGroup", + mock_headers, + newer_sha="ijkl789", + older_sha="mnop012", + ), + ] + + +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.requests.get" +) +def test_fetch_single_commit(mock_get): + mock_response = Mock() + mock_response.json.return_value = {"mock_response": "data"} + mock_get.return_value = mock_response + + actual = fetch_single_commit( + "mock_repo", "mock_owner", "mock_headers", "mock_sha" + ) + assert actual == {"mock_response": "data"} + mock_get.assert_called_once_with( + "https://api.github.com/repos/mock_owner/mock_repo/commits/mock_sha", + headers="mock_headers", + ) + + +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.requests.get" +) +def test_fetch_range_of_commits_results_on_first_page(mock_get): + mock_response_json_return_value = [ + {"sha": "tested_today"}, + {"sha": "intermediate_commit_1"}, + {"sha": "intermediate_commit_2"}, + {"sha": "tested_yesterday"}, + {"sha": "should_be_ignored"}, + ] + mock_response = Mock() + mock_response.json.return_value = mock_response_json_return_value + mock_get.return_value = mock_response + + mock_params = { + "per_page": 10, + "sha": "tested_today", + "page": 1, + } + actual = fetch_range_of_commits( + "mock_repo", + "mock_owner", + "mock_headers", + newer_sha="tested_today", + older_sha="tested_yesterday", + ) + assert actual == mock_response_json_return_value[:-1] + mock_get.assert_called_once_with( + "https://api.github.com/repos/mock_owner/mock_repo/commits/", + headers="mock_headers", + params=mock_params, + ) + + +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.requests.get" +) +def test_fetch_range_of_commits_results_on_later_page(mock_get): + mock_response_json_return_value = ( + [{"sha": "tested_today"}] + + [{"sha": f"intermediate_commit_{i}"} for i in range(1, 26)] + + [{"sha": "tested_yesterday"}] + + [{"sha": f"should_be_ignored_{i}"} for i in range(1, 21)] + ) + + mock_response = Mock() + mock_response.json.side_effect = [ + mock_response_json_return_value[:10], # First page + mock_response_json_return_value[10:20], # Second page + mock_response_json_return_value[20:30], # Third page + ] + mock_get.return_value = mock_response + + actual = fetch_range_of_commits( + "mock_repo", + "mock_owner", + "mock_headers", + newer_sha="tested_today", + older_sha="tested_yesterday", + ) + assert actual == mock_response_json_return_value[:27] + assert mock_get.call_count == 3 + + # TODO: Can't figure out why not working. + # expected_url = "https://api.github.com/repos/mock_owner/mock_repo/commits/" + # expected_params = { + # "per_page": 10, + # "sha": "tested_today", + # "page": 3, + # } + # expected_calls = [ + # call(expected_url, headers="mock_headers", params=expected_params), + # call(expected_url, headers="mock_headers", params=expected_params), + # call(expected_url, headers="mock_headers", params=expected_params), + # ] + # mock_get.assert_has_calls(expected_calls) + + +@pytest.mark.parametrize("num_of_commits_under_test", ["single", "range"]) +def test_process_commit_info(num_of_commits_under_test): + # Abbreviateded response for a real commit from ESMValTool. + mock_raw_commit = { + "sha": "662e792984d6577ae52e6931e794386ce508960c", + "commit": { + "author": { + "name": "github-actions[bot]", + "email": "41898282+github-actions[bot]@users.noreply.github.com", + "date": "2025-06-11T11:55:31Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-06-11T11:55:31Z", + }, + "message": "[Condalock] Update Linux condalock file (#4089)", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commit/" + "662e792984d6577ae52e6931e794386ce508960c", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/" + "662e792984d6577ae52e6931e794386ce508960c", + "author": { + "login": "github-actions[bot]", + "avatar_url": "https://avatars.githubusercontent.com/in/15368?v=4", + }, + "stats": {"total": 84, "additions": 42, "deletions": 42}, + "files": [], + } + mock_processed_commit = { + "sha": "662e792", + "author": "github-actions[bot]", + "message": "[Condalock] Update Linux condalock file (#4089)", + "date": "2025-06-11T11:55:31Z", + "url": "https://github.com/ESMValGroup/ESMValTool/commit/" + "662e792984d6577ae52e6931e794386ce508960c", + "author_avatar": "https://avatars.githubusercontent.com/in/15368?v=4", + } + + if num_of_commits_under_test == "single": + mock_commit_info = mock_raw_commit + expected = [mock_processed_commit] + else: + range_of_commits = 3 + mock_commit_info = [mock_raw_commit] * range_of_commits + expected = [mock_processed_commit] * range_of_commits + + actual = process_commit_info(mock_commit_info) + assert actual == expected diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3597a6ae5d..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[pycodestyle] -# ignore rules that conflict with ruff formatter -# E203: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices -# E501: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules -# W503: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes -ignore = E203,E501,W503 From a7b4a50f77dd2e2fec84743f8177d3afdc2c5b76 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 18 Jun 2025 14:51:35 +0100 Subject: [PATCH 31/40] #4036: Add docstrings to tests. Skips some tests while adding gh API --- .../bin/generate_html_report.py | 17 +++++------------ .../bin/test_fetch_commit_info.py | 14 ++++++++++++++ .../bin/test_generate_html_report.py | 18 ++++++++++++++++++ ...commits_via_git.py => test_shas_via_git.py} | 18 +++++++++++++++++- .../bin/test_shas_via_singularity.py | 10 ++++++++++ 5 files changed, 64 insertions(+), 13 deletions(-) rename esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/{test_commits_via_git.py => test_shas_via_git.py} (73%) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 1f875de85e..e6890c0ba3 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -16,7 +16,6 @@ fetch_commit_details_from_github_api, ) from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_git import ( - CommitInfo, get_shas_from_git, ) from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( @@ -311,7 +310,7 @@ def create_subheader(cylc_task_cycle_point): return subheader -def render_html_report(report_data, subheader, commit_info, sha_info): +def render_html_report(report_data, subheader, commit_info): """ Render the HTML report using Jinja2. @@ -321,19 +320,14 @@ def render_html_report(report_data, subheader, commit_info, sha_info): The report data to be rendered in the HTML template. subheader : str The subheader for the HTML report. - commit_info : CommitInfo | None - The commit information for ESMValCore and ESMValTool, if the site uses - git repos, or None. - sha_info : dict | None - The SHA information for ESMValCore and ESMValTool, if the site uses - singularity containers, or None. + commit_info : dict + The commit information for ESMValCore and ESMValTool. Returns ------- str The rendered HTML content. """ - commit_info = commit_info or CommitInfo([], []) script_dir = os.path.dirname(os.path.abspath(__file__)) env = Environment( loader=FileSystemLoader(script_dir), @@ -343,9 +337,8 @@ def render_html_report(report_data, subheader, commit_info, sha_info): rendered_html = template.render( subheader=subheader, report_data=report_data, - esmval_core_commits=commit_info.core, - esmval_tool_commits=commit_info.tool, - sha_info=sha_info, + esmval_core_commits=commit_info.get("ESMValCore", []), + esmval_tool_commits=commit_info.get("ESMValTool", []), ) return rendered_html diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py index ad202c79ca..ecdffc056a 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py @@ -1,3 +1,11 @@ +""" +Tests for fetching detailled commit information from the GitHub API. + +NOTE: The imports used mean these tests can only be run from within ESMValTool. +Ensure your ESMValTool working copy is installed in your environment before +running these tests. +""" + from unittest.mock import Mock, call, patch import pytest @@ -30,6 +38,7 @@ def test_fetch_commit_details_from_github_api_single_commits( mock_fetch_single, mock_shas_by_package_and_day ): + """Test correct function is called for fetching single commits.""" mock_fetch_single.return_value = {"mock_response": "data"} mock_headers = "headers" @@ -55,6 +64,7 @@ def test_fetch_commit_details_from_github_api_single_commits( def test_fetch_commit_details_from_github_api_range_of_commits( mock_fetch_range, ): + """Test correct function is called for fetching a range of commits.""" mock_fetch_range.return_value = [{"mock_response": "data"}] mock_headers = "headers" mock_shas_by_package_and_day = { @@ -94,6 +104,7 @@ def test_fetch_commit_details_from_github_api_range_of_commits( "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.requests.get" ) def test_fetch_single_commit(mock_get): + """Test fetching a single commit from the GitHub API.""" mock_response = Mock() mock_response.json.return_value = {"mock_response": "data"} mock_get.return_value = mock_response @@ -112,6 +123,7 @@ def test_fetch_single_commit(mock_get): "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.requests.get" ) def test_fetch_range_of_commits_results_on_first_page(mock_get): + """Test fetching a range of commits with results on the first page.""" mock_response_json_return_value = [ {"sha": "tested_today"}, {"sha": "intermediate_commit_1"}, @@ -147,6 +159,7 @@ def test_fetch_range_of_commits_results_on_first_page(mock_get): "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.requests.get" ) def test_fetch_range_of_commits_results_on_later_page(mock_get): + """Test fetching a range of commits with results on later pages.""" mock_response_json_return_value = ( [{"sha": "tested_today"}] + [{"sha": f"intermediate_commit_{i}"} for i in range(1, 26)] @@ -189,6 +202,7 @@ def test_fetch_range_of_commits_results_on_later_page(mock_get): @pytest.mark.parametrize("num_of_commits_under_test", ["single", "range"]) def test_process_commit_info(num_of_commits_under_test): + """Test processing commit information into a simplified format.""" # Abbreviateded response for a real commit from ESMValTool. mock_raw_commit = { "sha": "662e792984d6577ae52e6931e794386ce508960c", diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py index ff6853b1c9..e8d0a74b30 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py @@ -1,3 +1,11 @@ +""" +Tests for generating a HTML report from a Cylc database. + +NOTE: The imports used mean these tests can only be run from within ESMValTool. +Ensure your ESMValTool working copy is installed in your environment before +running these tests. +""" + import sqlite3 import tempfile from collections import namedtuple @@ -22,6 +30,7 @@ @pytest.fixture def mock_db_data_single_cycle(): + """Fixture providing a mock Cylc database with a single cycle of data.""" cycle = "20250521T0100Z" row_data = [ ("process_recipe_1", "succeeded", cycle), @@ -62,7 +71,9 @@ def mock_db_with_passed_values(row_data): yield path_to_synthetic_db +@pytest.mark.skip(reason="Needs reimplementation for GitHub API fetching") def test_main_for_site_dkrz(mock_db_data_single_cycle): + """Test the main function for a single cycle with DKRZ site.""" esmval_versions_today = mock_scm_version_output() # Duplicate input doesn't impact test. esmval_versions_yesterday = mock_scm_version_output() @@ -87,6 +98,7 @@ def test_main_for_site_dkrz(mock_db_data_single_cycle): def test_fetch_report_data_single_cycle(mock_db_data_single_cycle): + """Test fetching report data from a DB containing a single cycle's data.""" expected = [ ("process_recipe_1", "succeeded"), ("compare_recipe_1", "succeeded"), @@ -103,6 +115,7 @@ def test_fetch_report_data_single_cycle(mock_db_data_single_cycle): def test_fetch_report_data_multi_cycle(): + """Test fetching report data from a DB containing multiple cycles' data.""" mock_cycle = "20250521T0100Z" mock_cycle_minus_1d = "20250520T0100Z" mock_cycle_minus_2d = "20250519T1700Z" @@ -191,11 +204,13 @@ def test_fetch_report_data_multi_cycle(): ], ) def test_process_db_output(mock_db_output, expected): + """Test processing the DB output into a structured report data format.""" actual = process_db_output(mock_db_output) assert actual == expected def test_process_db_output_sorting(): + """Test that the DB output is sorted by recipe name.""" mock_db_output = [ ("process_c-ecipe", "succeeded"), ("compare_a-ecipe", "failed"), @@ -208,6 +223,7 @@ def test_process_db_output_sorting(): def test_create_subheader(): + """Test creating a subheader with a formatted cycle point.""" mock_cylc_task_cycle_point = "20250101T0001Z" actual = create_subheader(mock_cylc_task_cycle_point) assert actual == "Cycle start: 2025-01-01 00:01 UTC" @@ -215,6 +231,7 @@ def test_create_subheader(): @pytest.mark.skip(reason="Finish reimplementation") def test_render_html_report_no_commits_no_shas(): + """Test rendering an HTML report without commit SHAs.""" mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" mock_report_data = { "recipe_1": { @@ -236,6 +253,7 @@ def test_render_html_report_no_commits_no_shas(): @pytest.mark.skip(reason="Finish reimplementation") def test_render_html_report_partial(): + """Test rendering an HTML report with missing tasks.""" mock_subheader = "Cycle start: 2025-01-01 00:01 UTC" mock_report_data = { "recipe_1": { diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_git.py similarity index 73% rename from esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py rename to esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_git.py index cce556781b..7261aa7d3d 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_commits_via_git.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_git.py @@ -1,9 +1,21 @@ -from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_git import ( +""" +Tests for getting git commit SHAs via locally cloned git repositories. + +NOTE: The imports used mean these tests can only be run from within ESMValTool. +Ensure your ESMValTool working copy is installed in your environment before +running these tests. +""" + +import pytest + +from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.generate_html_report import ( add_report_messages_to_commits, ) +@pytest.mark.skip(reason="To be reimplemented") def test_add_report_message_to_git_commits_today_only(): + """Test adding report messages to git commits for today only.""" mock_commit_info = ( [{"date": "a", "sha": "abc", "author": "x", "message": "core"}], [{"date": "a", "sha": "xyz", "author": "y", "message": "tool"}], @@ -14,7 +26,9 @@ def test_add_report_message_to_git_commits_today_only(): assert mock_commit_info[1][0]["report_flag"] == expected +@pytest.mark.skip(reason="To be reimplemented") def test_add_report_message_to_git_commits_both_days(): + """Test adding report messages to git commits for today and yesterday.""" mock_commit_info = ( [{"date": "a", "sha": "123", "author": "x", "message": "core"}], [{"date": "a", "sha": "xyz", "author": "y", "message": "tool"}], @@ -25,7 +39,9 @@ def test_add_report_message_to_git_commits_both_days(): assert mock_commit_info[1][0]["report_flag"] == expected +@pytest.mark.skip(reason="To be reimplemented") def test_add_report_message_git_commits_both_days_multiple_commits(): + """Test adding report messages for both days with multiple commits.""" mock_commit_info = ( [ {"date": "a", "sha": "123", "author": "x", "message": "core_1"}, diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py index 79b2b79fcf..cfe88014ce 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_shas_via_singularity.py @@ -1,3 +1,11 @@ +""" +Tests for getting commit git SHAS from singularity containers. + +NOTE: The imports used mean these tests can only be run from within ESMValTool. +Ensure your ESMValTool working copy is installed in your environment before +running these tests. +""" + import pytest from esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.shas_via_singularity import ( @@ -48,6 +56,7 @@ def mock_scm_version_output(): def test_get_shas_from_singularity_and_validate_for_valid_shas( mock_day_version_today, mock_day_version_yesterday, expected ): + """Test getting SHAs from singularity and validating them.""" actual = get_shas_from_singularity( mock_day_version_today, mock_day_version_yesterday ) @@ -71,6 +80,7 @@ def test_get_shas_from_singularity_and_validate_for_valid_shas( ], ) def test_validate_all_shas_for_invalid_shas(shas, expected_message): + """Test validation of SHAs for missing or invalid values.""" with pytest.raises(ValueError, match=expected_message): # The unprocessed scm version strings are passed to the function purely # for error logging. Here 'None' is used. From e7d11483e673f56b1fff23c307d9c0c0184919cc Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Thu, 19 Jun 2025 16:53:17 +0100 Subject: [PATCH 32/40] #4036: MO working --- .../generate_report/bin/fetch_commit_info.py | 116 ++- .../bin/generate_html_report.py | 61 +- .../generate_report/bin/report_template.jinja | 89 +-- .../bin/shas_via_singularity.py | 3 +- .../app/generate_report/bin/test_data.py | 742 ++++++++++++++++++ .../bin/test_fetch_commit_info.py | 46 +- .../bin/test_generate_html_report.py | 3 + 7 files changed, 948 insertions(+), 112 deletions(-) create mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_data.py diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py index 3a574b5211..910e92feaf 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/fetch_commit_info.py @@ -1,13 +1,9 @@ -""" -Fetches commit details from the GitHub API. -""" +"""Fetch commit details from the GitHub API.""" import os import requests -tested_today = "662e792984d6577ae52e6931e794386ce508960c" - GITHUB_API_URL = "https://api.github.com" GITHUB_API_PERSONAL_ACCESS_TOKEN = os.environ.get( "GITHUB_API_PERSONAL_ACCESS_TOKEN" @@ -42,16 +38,25 @@ def fetch_commit_details_from_github_api( commit details for each day. E.g. {"ESMValCore": [{"sha": "abcd123", ...}, ...], "ESMValTool": [...]} """ + # Uncomment to test a range of commits during development. + # shas_by_package_and_day = { + # "ESMValCore": + # { + # "today": "d1453c665a97ad3563f9bf112274e273b9e4182b", + # "yesterday": "a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f"}, + # "ESMValTool": + # { + # "today": "c6e6e42d708265e3db1d57ccaa48669ca22011ca", + # "yesterday": "4515a2b9260ddef6b996a1d1b911e5d3d2029638"}, + # } commit_details_by_package = {} for package, shas_by_day in shas_by_package_and_day.items(): if shas_by_day.get("yesterday") is None or shas_by_day.get( "today" ) == shas_by_day.get("yesterday"): - raw_commit = fetch_single_commit( + raw_commits = fetch_single_commit( package, "ESMValGroup", headers, shas_by_day["today"] ) - # commit_info = process_commit_info(raw_commit) - commit_details_by_package[package] = [raw_commit] else: raw_commits = fetch_range_of_commits( package, @@ -60,11 +65,64 @@ def fetch_commit_details_from_github_api( newer_sha=shas_by_day["today"], older_sha=shas_by_day["yesterday"], ) - commit_details_by_package[package] = raw_commits - # commit_info = process_commit_info(commit_info) + commit_info = process_commit_info(raw_commits) + commit_details_by_package[package] = commit_info return commit_details_by_package +def make_api_call(url, headers=None, params=None): + """ + Make a GET request to a given API url. + + Parameters + ---------- + url : str + The URL to make the request to. + headers : dict, optional + Headers to include in the request. + params : dict, optional + Query parameters to include in the request. + + Raises + ------ + HTTPError + If the request fails or returns with a status code other than 200. + TimeOutError + If the request times out. + ConnectionError + If there is a connection error. + TooManyRequestsError + If the API rate limit is exceeded. + + Returns + ------- + Response + The raw response from the API call. + """ + try: + response = requests.get( + url, headers=headers, params=params, timeout=10 + ) + if response.status_code != 200: + raise requests.exceptions.HTTPError( + f"Unexpected status code for url={url} headers={headers} " + f"params={params} - {response.status_code}: {response.text}" + ) + except requests.exceptions.HTTPError as http_err: + raise requests.exceptions.HTTPError( + f"HTTP error occurred: {http_err}" + ) from http_err + except requests.exceptions.Timeout as timeout_err: + raise requests.exceptions.Timeout( + f"Request timed out: {timeout_err}" + ) from timeout_err + except requests.exceptions.ConnectionError as conn_err: + raise requests.exceptions.ConnectionError( + f"Connection error occurred: {conn_err}" + ) from conn_err + return response + + def fetch_single_commit(repo, owner, headers, sha): """ Fetch details of a single commit from the GitHub API. @@ -88,11 +146,10 @@ def fetch_single_commit(repo, owner, headers, sha): Returns ------- dict - The raw commit data if found, otherwise None. + The raw commit data if found. """ url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/commits/{sha}" - response = requests.get(url, headers=headers) - response.raise_for_status() # Raise a HTTP error for bad responses + response = make_api_call(url, headers=headers) raw_commit = response.json() return raw_commit @@ -106,13 +163,6 @@ def fetch_range_of_commits(repo, owner, headers, newer_sha, older_sha): to avoid hitting the API rate limits. NOTE: The GitHub API will raise a HTTPError if the newer SHA is not found. - Raises - ------ - HTTPError - If the newer SHA is not found (or if the request fails etc.) - ValueError - If too many pages are fetched, indicating a potential infinite loop. - Parameters: ---------- repo : str @@ -126,23 +176,34 @@ def fetch_range_of_commits(repo, owner, headers, newer_sha, older_sha): older_sha : str The SHA of the commit to stop fetching at. + Raises + ------ + HTTPError + If the newer SHA is not found (or if the request fails etc.) + ValueError + If too many pages are fetched, indicating a potential infinite loop. + + Returns + ------- + list[dict] + A list of raw commit data for the range of commits from newer_sha to + older_sha, in chronological order. """ - url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/commits/" + url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/commits" params = { "per_page": 10, "sha": newer_sha, } - range_raw_commits = [] page = 1 + range_raw_commits = [] fetched_end_sha = False + while not fetched_end_sha: params["page"] = page - response = requests.get(url, headers=headers, params=params) - response.raise_for_status() # Raise a HTTP error for bad responses + response = make_api_call(url, headers=headers, params=params) page_raw_commits = response.json() - for raw_commit in page_raw_commits: range_raw_commits.append(raw_commit) if raw_commit["sha"].startswith(older_sha): @@ -155,6 +216,7 @@ def fetch_range_of_commits(repo, owner, headers, newer_sha, older_sha): "Too many pages fetched, likely an infinite loop. Check the " "newer and older SHAs." ) + return range_raw_commits @@ -170,8 +232,8 @@ def process_commit_info(raw_commit_info): Returns ------- - dict - Processed commit information with relevant details. + list[dict] + A list of dictionaries containing processed commit information. """ if not isinstance(raw_commit_info, list): diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index e6890c0ba3..3b37a80714 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -8,7 +8,7 @@ from pathlib import Path from jinja2 import Environment, FileSystemLoader, select_autoescape -from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError, HTTPError, Timeout # Import from the ESMValTool package for testing. try: @@ -99,44 +99,63 @@ def main( # Catch the following errors so the report generates without commit/SHA # information. These errors are either propagated on purpose or # indicate a probable minor issue. - except (ValueError, KeyError, IndexError, HTTPError): + except ( + ValueError, + KeyError, + IndexError, + HTTPError, + Timeout, + ConnectionError, + ): print( "Report generating with results only. Error while fetching commit " "data. See std.err log for details." ) traceback.print_exc() + if sha_info: commit_info = fetch_commit_details_from_github_api(sha_info) - print(commit_info) + add_report_messages_to_commits(commit_info) - # raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) - # processed_db_data = process_db_output(raw_db_data) - # subheader = create_subheader(cylc_task_cycle_point) + raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) + processed_db_data = process_db_output(raw_db_data) + subheader = create_subheader(cylc_task_cycle_point) - # rendered_html = render_html_report( - # subheader=subheader, - # report_data=processed_db_data, - # commit_info=commit_info, - # ) + rendered_html = render_html_report( + subheader=subheader, + report_data=processed_db_data, + commit_info=commit_info, + ) - # write_report_to_file(rendered_html, report_path) + write_report_to_file(rendered_html, report_path) def add_report_messages_to_commits(commit_info): """ - Add report messages to a CommitInfo dataclass. + Add report messages to a commit info dictionary in-place. Parameters ---------- - commit_info : CommitInfo - A CommitInfo dataclass containing two lists of package commits. + commit_info : dict[str, list[dict]] + A dictionary where keys are package names and values are lists of + commit details. E.g. + { + "ESMValCore": [ + {"sha": "abcd123", "message": "Fix bug", ...}, + {"sha": "efgh456", "message": "Add feature", ...} + ], + "ESMValTool": [ + {"sha": "ijkl789", "message": "Update docs", ...} + ] + } """ - for package_commits in [commit_info.core, commit_info.tool]: - package_commits[0]["report_flag"] = "Version tested this cycle >>>" - if len(package_commits) > 1: - package_commits[-1]["report_flag"] = ( - "Version tested last cycle >>>" - ) + if commit_info: + for package_commits in commit_info.values(): + package_commits[0]["report_flag"] = "Commit tested this cycle >>>" + if len(package_commits) > 1: + package_commits[-1]["report_flag"] = ( + "Commit tested last cycle >>>" + ) def fetch_report_data(db_file_path, target_cycle_point): diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index b274c933e7..5ba701b26d 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -33,51 +33,27 @@ .table-full-width { width: 100%; + border-collapse: collapse; } - .table-half-width { - width: 50%; - } + .report-msg-column {width: 225px;} + .time-column {width: 75px;} + .sha-column {width: 50px;} + .author-column {width: 200px;} +

Recipe Test Workflow - Last Run Status

{{ subheader }}

- - {% if sha_info %} - - - - - - {%- if sha_info['ESMValCore'].get('yesterday') and sha_info['ESMValTool'].get('yesterday') -%} - - {%- endif -%} - - {%- for package, days in sha_info.items() -%} - - - - {%- if days.get('yesterday') -%} - - {%- endif -%} - - {%- endfor -%} -

Commit SHAs

Tested TodayTested Yesterday
{{ package }} - {{ days['today'] }} - - {{ days['yesterday'] }} -
- {% endif %} - {% if esmval_core_commits %} - - +

ESMValCore Commits

+ - - - - + + + + {%- for commit in esmval_core_commits -%} @@ -89,26 +65,30 @@ {%- endif -%} + - {%- endfor -%}

ESMValCore
TimeSHAAuthorTimeSHAAuthor Commit message
{{ commit['date'] }} - {{ commit['sha'] }} + {{ commit['sha'] }} + + Author Avatar + {{ commit['author'] }} {{ commit['author'] }} {{ commit['message']}}
{% endif %} - {% if esmval_tool_commits %} - - - - - - - - - - {%- for commit in esmval_tool_commits -%} +

ESMValTool Commits

TimeSHAAuthorCommit message
+ + + + + + + + + {%- for commit in esmval_tool_commits -%} {%- if commit['report_flag'] -%} @@ -116,8 +96,15 @@ {%- endif -%} - - + + {%- endfor -%} diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py index 85c975a6ec..1bbe0f2cf3 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py @@ -4,8 +4,7 @@ def get_shas_from_singularity(dev_versions_today, dev_versions_yesterday): - """ - Get git SHAs from ``setuptool-scm`` version strings. + """Get git SHAs from ``setuptool-scm`` version strings. SCM version strings are generated by the command ``esmvaltool version`` and imported as environment variables. Strings include both packages split by diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_data.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_data.py new file mode 100644 index 0000000000..9c25212828 --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_data.py @@ -0,0 +1,742 @@ +real_esmvalcore_raw_commit_info = [ + { + "sha": "d1453c665a97ad3563f9bf112274e273b9e4182b", + "node_id": "C_kwDOC1YaEdoAKGQxNDUzYzY2NWE5N2FkMzU2M2Y5YmYxMTIyNzRlMjczYjllNDE4MmI", + "commit": { + "author": { + "name": "Manuel Schlund", + "email": "32543114+schlunma@users.noreply.github.com", + "date": "2025-05-21T16:07:58Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-21T16:07:58Z", + }, + "message": "Always context managers when handling `netCDF4.Dataset` objects (#2734)", + "tree": { + "sha": "f1c57d4c1b0d118ed5f0c0cd5eb8ed66cb2bee94", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/trees/f1c57d4c1b0d118ed5f0c0cd5eb8ed66cb2bee94", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/commits/d1453c665a97ad3563f9bf112274e273b9e4182b", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoLfpeCRC1aQ7uu5UhlAAAcXEQAJ4AB6Kv8xv9M/DMKSIDqqbn\nMLFpZLjdLTNvX31tkVpJ3KFVlodYAocSsADwuJnAIrRjhNmhtrbJa/QOOdEhzEOS\n2/yjV7yB3r5NoFu6P0OsHPN2rTmih5QzlE0p3ONj0mNGul0gRaXLD0Ub+lS92hFf\nXhTFLDBk1N3FVwFtrx2Fh7xPf67LpWTimplbOEfjMzJhuovj5g5QLK1CTw4rFCYJ\nZhk437+i/kc1TBYFLE9s0JiedKYN1YCpmbX7vpeFFpX9efXaMUpJBMNrB7qwrQLV\n9QjTyfpdn6FzD2/VKYzUdFEfa4j7y/VEyP9oLSn7Cyu1pU83W/lEmHAFixmJi5j4\nocfVP8C3Lb6mE65rrBVIZdhL8KM+1ocg9OTQzB0XCr1qQbJjKTWQJTSuM9QM77Dc\ntFxMY107Xu+ESq9wwWFZpOgmy8Ji3CjoLDoMgKomEV1K17M5uHFftc8xVzkWRs0z\nAQYaVLjaoUobOgilmtu8mDQO/leg1SIs9b+yAJ02vhHuHOiAcKbwslrrnVpeU8HQ\nNyXLxp6QDrGzar/a65BJTt4hNWfpQ2xB67NONoPuC+w63fTCV2j1QUTokjbOAaLo\n/EW1QrqPwjP1FJ+tyfbNxTJje1CNNhu0ITOTQlb39gA0SU7LaZNdNavN+TOs7pKj\nPSTrRwB/bQmd7int3+yu\n=0pb5\n-----END PGP SIGNATURE-----\n", + "payload": "tree f1c57d4c1b0d118ed5f0c0cd5eb8ed66cb2bee94\nparent 4d330234ed6f97c4a53939e28486d3cece285e7b\nauthor Manuel Schlund <32543114+schlunma@users.noreply.github.com> 1747843678 +0200\ncommitter GitHub 1747843678 +0100\n\nAlways context managers when handling `netCDF4.Dataset` objects (#2734)\n\n", + "verified_at": "2025-05-21T16:08:00Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/d1453c665a97ad3563f9bf112274e273b9e4182b", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/d1453c665a97ad3563f9bf112274e273b9e4182b", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/d1453c665a97ad3563f9bf112274e273b9e4182b/comments", + "author": { + "login": "schlunma", + "id": 32543114, + "node_id": "MDQ6VXNlcjMyNTQzMTE0", + "avatar_url": "https://avatars.githubusercontent.com/u/32543114?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/schlunma", + "html_url": "https://github.com/schlunma", + "followers_url": "https://api.github.com/users/schlunma/followers", + "following_url": "https://api.github.com/users/schlunma/following{/other_user}", + "gists_url": "https://api.github.com/users/schlunma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/schlunma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/schlunma/subscriptions", + "organizations_url": "https://api.github.com/users/schlunma/orgs", + "repos_url": "https://api.github.com/users/schlunma/repos", + "events_url": "https://api.github.com/users/schlunma/events{/privacy}", + "received_events_url": "https://api.github.com/users/schlunma/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "4d330234ed6f97c4a53939e28486d3cece285e7b", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/4d330234ed6f97c4a53939e28486d3cece285e7b", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/4d330234ed6f97c4a53939e28486d3cece285e7b", + } + ], + }, + { + "sha": "4d330234ed6f97c4a53939e28486d3cece285e7b", + "node_id": "C_kwDOC1YaEdoAKDRkMzMwMjM0ZWQ2Zjk3YzRhNTM5MzllMjg0ODZkM2NlY2UyODVlN2I", + "commit": { + "author": { + "name": "pre-commit-ci[bot]", + "email": "66853113+pre-commit-ci[bot]@users.noreply.github.com", + "date": "2025-05-21T16:03:16Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-21T16:03:16Z", + }, + "message": "[pre-commit.ci] pre-commit autoupdate (#2735)\n\nCo-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>", + "tree": { + "sha": "2327d96f92e8236c6f3155b1bad9ea4319d47141", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/trees/2327d96f92e8236c6f3155b1bad9ea4319d47141", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/commits/4d330234ed6f97c4a53939e28486d3cece285e7b", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoLflECRC1aQ7uu5UhlAAAfCAQAJjGIewykXxNXE7j5iYqBW/A\nVMJKrGlqQXA2kCn/J93gHm/1MuR4nAalYjRMQ/ajr38xWC//yyQMGnHOrAB6820T\nIDnLqyG0rDwtRh7T2X+f2w7KyNH9gFXPkcoP+VA4AOpPfLvDy4fp7gb29H+CRCO0\nf6gZNXpcH+Mj3pE4mIJfH7Ie0wA71ZKwNdiSxYgQyGEL77LhsvndY+IpZjFshB1m\nJS9A6jfT0IiMAzjAawDbgN6RaP0Sugh91fsc56t32DifFRVxp/6w80Isi+r/uj1D\nRUziw/fmvR+8qwm3818U5llfkoKlwzsgfU8nsfRjkDzqRENpY4hAvGJjSYT6Cq29\nqTqL32TTDsdofX/NZ01dooQmHAgbIFHN6EcOimib+QT8S0Qg2lCxOPiZmFSqnQqi\nr0E+1mMY6jAC5zdpC7HwiTPGqr9p3hQL7KDP1tRcjObVFZq7naKflIJhp3wWfVC7\ns2pta1Cy2D1RB1WzCMIXT65NG4P9hGVm0S6O8m0muNsIjbq27QtJzllfDdRmClUi\nWQkTomaAlrR96z3tp9k32KO4EVMOhOk3hQtpAju1/D4ClGqYGjG3WOg4tmaxf9Ag\nGc3lPG/LATzAve3vwv6pEdPWNhxVpq8ClG6ECHyXnipeBGgljl/xG65c7ERuCukL\nD08rcmIo/fMvf81RGkzo\n=mFhk\n-----END PGP SIGNATURE-----\n", + "payload": "tree 2327d96f92e8236c6f3155b1bad9ea4319d47141\nparent d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e\nauthor pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> 1747843396 +0100\ncommitter GitHub 1747843396 +0100\n\n[pre-commit.ci] pre-commit autoupdate (#2735)\n\nCo-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>", + "verified_at": "2025-05-21T16:03:19Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/4d330234ed6f97c4a53939e28486d3cece285e7b", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/4d330234ed6f97c4a53939e28486d3cece285e7b", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/4d330234ed6f97c4a53939e28486d3cece285e7b/comments", + "author": { + "login": "pre-commit-ci[bot]", + "id": 66853113, + "node_id": "MDM6Qm90NjY4NTMxMTM=", + "avatar_url": "https://avatars.githubusercontent.com/in/68672?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D", + "html_url": "https://github.com/apps/pre-commit-ci", + "followers_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/pre-commit-ci%5Bbot%5D/received_events", + "type": "Bot", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + } + ], + }, + { + "sha": "d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + "node_id": "C_kwDOC1YaEdoAKGQwM2Q5MWNlZTJmZjhiNzVmMzZkNWVjOGYwYTc5ZjMxZmMzMzkwNWU", + "commit": { + "author": { + "name": "github-actions[bot]", + "email": "41898282+github-actions[bot]@users.noreply.github.com", + "date": "2025-05-21T16:02:38Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-21T16:02:38Z", + }, + "message": "[Condalock] Update Linux condalock file (#2737)\n\nCo-authored-by: valeriupredoi ", + "tree": { + "sha": "73848b293c70f7dafd83e240aa5795689e6cc718", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/trees/73848b293c70f7dafd83e240aa5795689e6cc718", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/commits/d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoLfkeCRC1aQ7uu5UhlAAA9/4QAHINkHRiUSMVq5j8HQAPCyl7\nlcF8W5ZVi+CDLvpEJQuZd+kZzut3A4ZLJLsTdrmnraIwOL7jK/7hAEeqlQehAcPq\n/RWahIOMMTkle+bV4+LvVlm3Mv+i7ELQZW3LKcA7lnz5YlKhO1Cwc4D5wv35MfGw\n7VrxGdrcP3VYrybn2XZE7PQcEeeFrkT38YcrCkegD1X27E9ikaYoCEcVDdbj8AVm\nRUvt+9MHNrqlsfNDGRQHg1NnEm7h/lzHXPM+dN6Z1kIdVG37o887+lPE57UO+N5S\ne3EnGBBJFmVx4PjqSw/0AE0bMoXkIdQcUffxRRRARsi/FT6vinjhoLdhe4nu2S0P\nJb5OvAP89+3N1XDXutmWqHzaesTT7iYu4/LQ3lu5axIU6lG+z2MwzrA21AARWmlZ\nHY/aFDUziSbFeOd1ddPPXcv6kclLcAdaPjg5Uqev6AELcHc854Yo6Kmz9T5C/9e4\nu5ykyskq04Mop9h8o2m7eKJJiFbGzmQnTjtghi2kF5bkOLQ/oACRrcvhc9nSGLJr\ndkz4sWOk87LyAJxeMFeSHc7h+6mWFpsU5VEi3cuh+A75HbsecjGHfM/mpOecL2qD\nwipn/9Doe6OnhQ7nsihKgKjHKh5mq6puAUqrbxQGGf2tE3FWBdSZE9RCde5JErJA\nQ7FMncXdXkRKHaGqLJcv\n=W94Y\n-----END PGP SIGNATURE-----\n", + "payload": "tree 73848b293c70f7dafd83e240aa5795689e6cc718\nparent a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f\nauthor github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> 1747843358 +0100\ncommitter GitHub 1747843358 +0100\n\n[Condalock] Update Linux condalock file (#2737)\n\nCo-authored-by: valeriupredoi ", + "verified_at": "2025-05-21T16:02:42Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/d03d91cee2ff8b75f36d5ec8f0a79f31fc33905e/comments", + "author": { + "login": "github-actions[bot]", + "id": 41898282, + "node_id": "MDM6Qm90NDE4OTgyODI=", + "avatar_url": "https://avatars.githubusercontent.com/in/15368?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-actions%5Bbot%5D", + "html_url": "https://github.com/apps/github-actions", + "followers_url": "https://api.github.com/users/github-actions%5Bbot%5D/followers", + "following_url": "https://api.github.com/users/github-actions%5Bbot%5D/following{/other_user}", + "gists_url": "https://api.github.com/users/github-actions%5Bbot%5D/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-actions%5Bbot%5D/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-actions%5Bbot%5D/subscriptions", + "organizations_url": "https://api.github.com/users/github-actions%5Bbot%5D/orgs", + "repos_url": "https://api.github.com/users/github-actions%5Bbot%5D/repos", + "events_url": "https://api.github.com/users/github-actions%5Bbot%5D/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-actions%5Bbot%5D/received_events", + "type": "Bot", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + } + ], + }, + { + "sha": "a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + "node_id": "C_kwDOC1YaEdoAKGExMTZjYzZhNjBhNmFlOGI2MjhlMGZmY2JjODk0Mjk0Y2QyNmVlNWY", + "commit": { + "author": { + "name": "Manuel Schlund", + "email": "32543114+schlunma@users.noreply.github.com", + "date": "2025-05-16T10:41:33Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-16T10:41:33Z", + }, + "message": "Allow reading facets from filenames (#2725)", + "tree": { + "sha": "8ec4cf79a1b35858a2159cd33b4818b4b1656bc6", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/trees/8ec4cf79a1b35858a2159cd33b4818b4b1656bc6", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/git/commits/a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoJxZdCRC1aQ7uu5UhlAAAm+4QAIeKMdmAAgT1Nn0AF00+aOWx\nh/WWuicLqTsJZ9FRHYTYSH1KBpF3O86JzvgHGWmfmaoH9VyDevL4LrjwhQHuI/tS\n9LKHn8jIUaomBD4Iglu1IckI63QwgeaT+UQbPw8sppcc0xXD7c9odJ8vwQCds09z\n+MLWOwfmbSVe1psknVuGh4f74sPIofbRD3DghRpm59NlfFSflHNFOH9jzFxg8hNA\n2ZXovrrmZER4va7somm+XgqOkERjXJuJP1WxK1DzJrptDCK8TSRyqyi3LH9h5Nl3\nsfn5Ack6YZrXi4BAP3A2E0Kp6uZF1AeF9FxycMFWbEJ86/OQZBa0ti5L06b8A2mF\nFJZlku2dVASDZCuf6wBOS4bw7fp1Fj3D2rJrmV89dSPtgsFvSILKZTF1gxRhn9iz\nj/n1M3wIHVBObclRwvE51MIOJGHTeoSKKqzEK+FglJ7HB5zm0hHkaHZSQr2W/rBb\n0s85T1Gc4NxSqbfyw3ZBh3/VdwwJg+oWz8dKBJCKS1Ppkbsbj122PtDBPaeO9B77\nUEDY7FDo+MH1qOjRPIP5RYTazkqnjClGEmhgBeAc/uPioN73LKWN0Eq7ozpEZNU6\nRfEEfSNCPFBq5rkc5jfDZ3ZkoQWK4d43QAEPE7pCtClyo8zlmmENmLoxCdx/68TW\nEELsUJSCzKojU+fNS9J0\n=uPvX\n-----END PGP SIGNATURE-----\n", + "payload": "tree 8ec4cf79a1b35858a2159cd33b4818b4b1656bc6\nparent 170a938935fa39aa82abc004b9c298e2e977fa7e\nauthor Manuel Schlund <32543114+schlunma@users.noreply.github.com> 1747392093 +0200\ncommitter GitHub 1747392093 +0200\n\nAllow reading facets from filenames (#2725)\n\n", + "verified_at": "2025-05-16T10:41:37Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/a116cc6a60a6ae8b628e0ffcbc894294cd26ee5f/comments", + "author": { + "login": "schlunma", + "id": 32543114, + "node_id": "MDQ6VXNlcjMyNTQzMTE0", + "avatar_url": "https://avatars.githubusercontent.com/u/32543114?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/schlunma", + "html_url": "https://github.com/schlunma", + "followers_url": "https://api.github.com/users/schlunma/followers", + "following_url": "https://api.github.com/users/schlunma/following{/other_user}", + "gists_url": "https://api.github.com/users/schlunma/gists{/gist_id}", + "starred_url": "https://api.github.com/users/schlunma/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/schlunma/subscriptions", + "organizations_url": "https://api.github.com/users/schlunma/orgs", + "repos_url": "https://api.github.com/users/schlunma/repos", + "events_url": "https://api.github.com/users/schlunma/events{/privacy}", + "received_events_url": "https://api.github.com/users/schlunma/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "170a938935fa39aa82abc004b9c298e2e977fa7e", + "url": "https://api.github.com/repos/ESMValGroup/ESMValCore/commits/170a938935fa39aa82abc004b9c298e2e977fa7e", + "html_url": "https://github.com/ESMValGroup/ESMValCore/commit/170a938935fa39aa82abc004b9c298e2e977fa7e", + } + ], + }, +] +real_esmvaltool_raw_commit_info = [ + { + "sha": "c6e6e42d708265e3db1d57ccaa48669ca22011ca", + "node_id": "C_kwDOBMaKjdoAKGM2ZTZlNDJkNzA4MjY1ZTNkYjFkNTdjY2FhNDg2NjljYTIyMDExY2E", + "commit": { + "author": { + "name": "Emma Hogan", + "email": "ehogan@users.noreply.github.com", + "date": "2025-05-16T14:34:51Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-16T14:34:51Z", + }, + "message": '#4041: Revert "Update RTW to use the `git` scheme"', + "tree": { + "sha": "061101f377491a5fc0e0b12ffd5cf872a31aea7c", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/trees/061101f377491a5fc0e0b12ffd5cf872a31aea7c", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/commits/c6e6e42d708265e3db1d57ccaa48669ca22011ca", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoJ00LCRC1aQ7uu5UhlAAAhFAQAAMSqXR2BG+G95U9HhyhtZ3c\nSWmDuVKtJWmItQsAV8wapGhw7PCXxJ/mblPf6PZ7r7Q12UMlHaKsm8y7QS7Sd32y\n3TvwKCKXkr9CnSYJcVDxL3uQ0Fg7sKbrKMGzPMpYRDAjpJPGjSyEdj1N8jVq19YO\nQYfsbF2PjGyYMvpBfo8qfLQ4nfVfC1UZheLxILSmdSTM8l7fgQ8Lj91vnivxmXuo\nP1lKe2NDhdIEGtcdspLCMqMsvJR/Q9wmXZ8gyRL+mUVzQPkj/Lo3EmdDB9aIQAXZ\n1lko9S9xwJJJcAV8EyvXu+/+Pcgn6FobfWmVTRk2EFAt2ntCfHY9Dux6d6h3r1i7\nvyF8Vn+Zutrzj8mZ2atXnfDhFC851WgdKjXbTSGjZe9pnbwMLPZOIVwQ6WueiaRz\nDAysVW7uS11egd3qkolCRmBWGFAjGEXcMwhXNhokB/yVOdrN9IRNmE0LJ83GJHCc\nE6YlLEBwXZl9OO7jngsPFt3QexyQRvEGliwY+rqGEPlvEfjUnqcVF+L27j85PjZS\nzbGe1g2c5ox1p5bmJiQe2l3XxI8KQAxtm1Cp9UQSpjA8dw6COjhAraLL1OefTq3z\nTVEoyDZYQM6ovNkActsI0wbYQRFjCXG7WYV1nGHgSHid+gayBXRmuiACEL8l8vOv\nRiXgjFtSeo1JU8HON13a\n=FWnv\n-----END PGP SIGNATURE-----\n", + "payload": 'tree 061101f377491a5fc0e0b12ffd5cf872a31aea7c\nparent e92a0ed22345e56f26b30c7c004620cd04ca2535\nauthor Emma Hogan 1747406091 +0100\ncommitter GitHub 1747406091 +0100\n\n#4041: Revert "Update RTW to use the `git` scheme"\n\n', + "verified_at": "2025-05-16T14:39:54Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/c6e6e42d708265e3db1d57ccaa48669ca22011ca", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/c6e6e42d708265e3db1d57ccaa48669ca22011ca", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/c6e6e42d708265e3db1d57ccaa48669ca22011ca/comments", + "author": { + "login": "ehogan", + "id": 918142, + "node_id": "MDQ6VXNlcjkxODE0Mg==", + "avatar_url": "https://avatars.githubusercontent.com/u/918142?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ehogan", + "html_url": "https://github.com/ehogan", + "followers_url": "https://api.github.com/users/ehogan/followers", + "following_url": "https://api.github.com/users/ehogan/following{/other_user}", + "gists_url": "https://api.github.com/users/ehogan/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ehogan/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ehogan/subscriptions", + "organizations_url": "https://api.github.com/users/ehogan/orgs", + "repos_url": "https://api.github.com/users/ehogan/repos", + "events_url": "https://api.github.com/users/ehogan/events{/privacy}", + "received_events_url": "https://api.github.com/users/ehogan/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "e92a0ed22345e56f26b30c7c004620cd04ca2535", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/e92a0ed22345e56f26b30c7c004620cd04ca2535", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/e92a0ed22345e56f26b30c7c004620cd04ca2535", + } + ], + }, + { + "sha": "e92a0ed22345e56f26b30c7c004620cd04ca2535", + "node_id": "C_kwDOBMaKjdoAKGU5MmEwZWQyMjM0NWU1NmYyNmIzMGM3YzAwNDYyMGNkMDRjYTI1MzU", + "commit": { + "author": { + "name": "Emma Hogan", + "email": "ehogan@users.noreply.github.com", + "date": "2025-05-16T10:13:59Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-16T10:13:59Z", + }, + "message": "#4019: Add housekeeping to RTW", + "tree": { + "sha": "bd2aeac4fcd48ec002d93407afe2bb08c63661a0", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/trees/bd2aeac4fcd48ec002d93407afe2bb08c63661a0", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/commits/e92a0ed22345e56f26b30c7c004620cd04ca2535", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoJw/nCRC1aQ7uu5UhlAAAyE4QAJYyxcguy1ZSNAm0NOF7SlT/\nxWZ1BBziFc70hqMh/ZykkUmDwxKa7J2vwUibCbov2ctflcLjKOlGdZtU//Sb341q\nfh2GrSIkRISr9VX0TnI21ej7HyuhUE5xyq5gJm/w4AnxZ40PuZGmUlw1IbyRfQHO\nB3gga/WYHADPSaRQshVrE822OpPy7k1cwsezrCBef9OGeL1q9RS9eoD43VtoLawG\n0wusmuBXy4eLI07tbBnigXTyhagBXyt/JmIndZ/CNcnBr8ess/atGS3xZpve0m+q\no74bDG3OB7x9ZRLilr2iqgbYYiDWzAYY/Mv5qDQySRqNrGMoVWxGTHlflOMmxX27\nWJiLl75U06ZoaEPTgwIqg+rqqrMO70Pgo8f0MNzhbbtCdZVyQb8lKpCmMfV/KfkP\nhtU3vW9UhtvOBZbdFqlV8E892NM9u7guLmUoPa07wTFjtoFEGVGfeySB0FeHGDNb\nObTFx6RqQ2csiZwsWICb0kC+Gj39Zuw7F1BaDqwdN82NqWmrJwHjkGK2CgYNWIh1\nb8DhLONwTnRxy7Ip5qSXyr7Uh4Mb/Ec3jFtoWSN3DDFJtPkdiMywq8FbEfgP83mH\n4QP36hUR+QynHn2nzUyAJUfOoe83sxu4PiEUUb1cBoJTSBJa4fx6Z0bRpBmw5nsx\nCUX1GdxIupDL4W054u8g\n=1bXy\n-----END PGP SIGNATURE-----\n", + "payload": "tree bd2aeac4fcd48ec002d93407afe2bb08c63661a0\nparent cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed\nauthor Emma Hogan 1747390439 +0100\ncommitter GitHub 1747390439 +0100\n\n#4019: Add housekeeping to RTW\n\n", + "verified_at": "2025-05-16T10:19:04Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/e92a0ed22345e56f26b30c7c004620cd04ca2535", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/e92a0ed22345e56f26b30c7c004620cd04ca2535", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/e92a0ed22345e56f26b30c7c004620cd04ca2535/comments", + "author": { + "login": "ehogan", + "id": 918142, + "node_id": "MDQ6VXNlcjkxODE0Mg==", + "avatar_url": "https://avatars.githubusercontent.com/u/918142?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ehogan", + "html_url": "https://github.com/ehogan", + "followers_url": "https://api.github.com/users/ehogan/followers", + "following_url": "https://api.github.com/users/ehogan/following{/other_user}", + "gists_url": "https://api.github.com/users/ehogan/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ehogan/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ehogan/subscriptions", + "organizations_url": "https://api.github.com/users/ehogan/orgs", + "repos_url": "https://api.github.com/users/ehogan/repos", + "events_url": "https://api.github.com/users/ehogan/events{/privacy}", + "received_events_url": "https://api.github.com/users/ehogan/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + } + ], + }, + { + "sha": "cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + "node_id": "C_kwDOBMaKjdoAKGNkZDI3OGI2ZDBjYmRmZTA5OGY1NmVlMTRlZjhhNzVjZGI1YWIxZWQ", + "commit": { + "author": { + "name": "Emma Hogan", + "email": "ehogan@users.noreply.github.com", + "date": "2025-05-16T09:53:26Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-16T09:53:26Z", + }, + "message": "#4030: Update RTW to use the `git` scheme\n\nCo-authored-by: chrisbillowsMO <152496175+chrisbillowsMO@users.noreply.github.com>", + "tree": { + "sha": "ad66eb0848a2e80d5639ce55546b53fe76c173df", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/trees/ad66eb0848a2e80d5639ce55546b53fe76c173df", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/commits/cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoJwsWCRC1aQ7uu5UhlAAAtI0QACf7iLFICYoJ6d8kVnjCl5ck\n6D7EvfJDk5mO2bCFeA4Iy/SIBAYWpM44GKFRvu1Ikkx6Zmbwgdj0UGUDj0wsDMjp\nLAQLGCG7VRVWZUKSQ4MdJwf4vcpn9zgPDjTPx6k/kzVgcKtJKzoDbU4u2EulHcwr\n95IjEiB9CwkC4Aq7h1Zh/1IE7Zr2U2O1sBiMjvSNk3wFGZ0p2vS4RtozaBLGsGfB\nBjv7cIRfH3EszrlItIFPjIKRdipoliXuBgrjH9AO6t8kyF9uvKpg6igejAS/Z14N\nKFhJtkdjoEACrIVuRAxpkdKavdueiWWO4EQFSMrZY2nwUpnuIug/0qRzrze/YcB2\n4CLFxCV53qtHbjz3pHw1wieHhEqLkJ1jkD7JgWoQqW3b9PLLqztHrWKyI0WUw40j\nLIXhwoJ6T6jaReIgbOOi5IqNHBdi/xJaikXa9/4jjk21GRhmDCYXejdoAWb+3Gii\n827JBiW1h6rF6UnMOhViWlorWU/YPA3rMZbNEHMN4cgZ//jbM6SyTUqbbdeN/W4k\n3PQZ9Zbmu8tg+bSK4V24xZ2PZQgZlo+zE3NHIwJPPri5aWOtKaTmsjR7IStUXRBr\nU8P46QoinzzNvyxAcCMYVm//VSn7O/8f7HomlE/nUyPvzBd3ZnZG9GpXu2f8TaOO\nLM4Bd2icdanFayKNudTB\n=ckQG\n-----END PGP SIGNATURE-----\n", + "payload": "tree ad66eb0848a2e80d5639ce55546b53fe76c173df\nparent 9cfe504a84da028320a622e75d8fdbb43be0e475\nauthor Emma Hogan 1747389206 +0100\ncommitter GitHub 1747389206 +0100\n\n#4030: Update RTW to use the `git` scheme\n\nCo-authored-by: chrisbillowsMO <152496175+chrisbillowsMO@users.noreply.github.com>", + "verified_at": "2025-05-16T09:54:47Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/cdd278b6d0cbdfe098f56ee14ef8a75cdb5ab1ed/comments", + "author": { + "login": "ehogan", + "id": 918142, + "node_id": "MDQ6VXNlcjkxODE0Mg==", + "avatar_url": "https://avatars.githubusercontent.com/u/918142?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ehogan", + "html_url": "https://github.com/ehogan", + "followers_url": "https://api.github.com/users/ehogan/followers", + "following_url": "https://api.github.com/users/ehogan/following{/other_user}", + "gists_url": "https://api.github.com/users/ehogan/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ehogan/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ehogan/subscriptions", + "organizations_url": "https://api.github.com/users/ehogan/orgs", + "repos_url": "https://api.github.com/users/ehogan/repos", + "events_url": "https://api.github.com/users/ehogan/events{/privacy}", + "received_events_url": "https://api.github.com/users/ehogan/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "9cfe504a84da028320a622e75d8fdbb43be0e475", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/9cfe504a84da028320a622e75d8fdbb43be0e475", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/9cfe504a84da028320a622e75d8fdbb43be0e475", + } + ], + }, + { + "sha": "9cfe504a84da028320a622e75d8fdbb43be0e475", + "node_id": "C_kwDOBMaKjdoAKDljZmU1MDRhODRkYTAyODMyMGE2MjJlNzVkOGZkYmI0M2JlMGU0NzU", + "commit": { + "author": { + "name": "chrisbillowsMO", + "email": "152496175+chrisbillowsMO@users.noreply.github.com", + "date": "2025-05-16T09:28:07Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-16T09:28:07Z", + }, + "message": '#4013: Generate "recipe status" HTML after each Recipe Test Workflow cycle\n\nCo-authored-by: Emma Hogan ', + "tree": { + "sha": "4bad0a6224d1b3fcb54cdaca1e2c58268f65843d", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/trees/4bad0a6224d1b3fcb54cdaca1e2c58268f65843d", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/commits/9cfe504a84da028320a622e75d8fdbb43be0e475", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoJwUnCRC1aQ7uu5UhlAAAuAAQAKdDypCKGe1tAQo3cAROWVYi\n8iHS6vm0++6Bg/178T7chdnuLYVs7F7qSbGFe4UVzFg06dvTPn84hY5FFF4K+nkG\nZA5Aaw+HwfXCXZF1hJHsPAWvWM0FR3CxjZK0NEoCuWqTbEgEXokvC1+H/c5uvdi+\nJ+VBoIUZyW6CL3PwcUUadIeEyC6OXvyiu88momQrn6dpSIMBgLJxdRhE62TaeSkD\nAGvEGb2U1N2V6ZSeiZojesWWQM+dk+P1zW4puFSw04IJftxj2mYecpf/cT0abwHH\n/z9kmzRHd31tV+C9aNrUB2bJSmrhZsSJ5Lj8Gaz9VwIm4/j2Scrb/M1bdTORkwEx\nNCuwa4Q54MC8u2kIcswbOo8hJf/X8owdpxuwuDc2troke1TAr37xMS3MaeSiuiGH\nraVgN6o6oIHnIlunWrnTQEALZSLppmpnt/cCyEL38yNOiG24F0xmlznaeCuUifeh\nJ4Aybblv0drwI2iM/R9M3fRVzePM6bHEVvGH19aaDgyt+NM30gUmjKaraQ0BbEAa\nlZNIhlXlI7GGwkFSbbGpigkiwQcEi0014bER34A0hWafGOfaY6ZjQ7+7WXZYcSp6\noUpOYtsUqKPXKoB8Sdc8KcJYR5kQ1KK+yKkLpNxcxPInrX9DuWNlsPm0xFwgmt+8\nfLY6k+KzDsV/SUexh4FX\n=d2DF\n-----END PGP SIGNATURE-----\n", + "payload": 'tree 4bad0a6224d1b3fcb54cdaca1e2c58268f65843d\nparent 4515a2b9260ddef6b996a1d1b911e5d3d2029638\nauthor chrisbillowsMO <152496175+chrisbillowsMO@users.noreply.github.com> 1747387687 +0100\ncommitter GitHub 1747387687 +0100\n\n#4013: Generate "recipe status" HTML after each Recipe Test Workflow cycle\n\nCo-authored-by: Emma Hogan ', + "verified_at": "2025-05-16T09:30:04Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/9cfe504a84da028320a622e75d8fdbb43be0e475", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/9cfe504a84da028320a622e75d8fdbb43be0e475", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/9cfe504a84da028320a622e75d8fdbb43be0e475/comments", + "author": { + "login": "chrisbillowsMO", + "id": 152496175, + "node_id": "U_kgDOCRboLw", + "avatar_url": "https://avatars.githubusercontent.com/u/152496175?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/chrisbillowsMO", + "html_url": "https://github.com/chrisbillowsMO", + "followers_url": "https://api.github.com/users/chrisbillowsMO/followers", + "following_url": "https://api.github.com/users/chrisbillowsMO/following{/other_user}", + "gists_url": "https://api.github.com/users/chrisbillowsMO/gists{/gist_id}", + "starred_url": "https://api.github.com/users/chrisbillowsMO/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/chrisbillowsMO/subscriptions", + "organizations_url": "https://api.github.com/users/chrisbillowsMO/orgs", + "repos_url": "https://api.github.com/users/chrisbillowsMO/repos", + "events_url": "https://api.github.com/users/chrisbillowsMO/events{/privacy}", + "received_events_url": "https://api.github.com/users/chrisbillowsMO/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "4515a2b9260ddef6b996a1d1b911e5d3d2029638", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/4515a2b9260ddef6b996a1d1b911e5d3d2029638", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/4515a2b9260ddef6b996a1d1b911e5d3d2029638", + } + ], + }, + { + "sha": "4515a2b9260ddef6b996a1d1b911e5d3d2029638", + "node_id": "C_kwDOBMaKjdoAKDQ1MTVhMmI5MjYwZGRlZjZiOTk2YTFkMWI5MTFlNWQzZDIwMjk2Mzg", + "commit": { + "author": { + "name": "Alistair Sellar", + "email": "16133375+alistairsellar@users.noreply.github.com", + "date": "2025-05-15T16:26:38Z", + }, + "committer": { + "name": "GitHub", + "email": "noreply@github.com", + "date": "2025-05-15T16:26:38Z", + }, + "message": "Remove `test_recipe` command (#4031)", + "tree": { + "sha": "79ed9f0356648c142701f10ea04f2dcf9b87385d", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/trees/79ed9f0356648c142701f10ea04f2dcf9b87385d", + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/git/commits/4515a2b9260ddef6b996a1d1b911e5d3d2029638", + "comment_count": 0, + "verification": { + "verified": True, + "reason": "valid", + "signature": "-----BEGIN PGP SIGNATURE-----\n\nwsFcBAABCAAQBQJoJhW+CRC1aQ7uu5UhlAAAayYQADngmuwCYsadY38cFzCTsGhI\nI8dSRNBOZsgoukAttQDZp28+FQ5C9R0dKvJzILXTepKPLSNZEhqBbtFGISkpkSAQ\n7RFwa3yKcyqH+xkYedddR+5Abd+u/8IVs7IKqdNU1VKodLv7PW0yvLpVDPHzZSiV\nEkneXWvzGesbsc8AIbAZUqR/ZzGaVWB25LCSRXS+rZLY/PVpWII30mTy4UHTEEbc\ntMHmWwzpXK832yORqZd46igCT4fnIqZxtMC/JbpSt8d3MjqdIXw6iSvATyXvgtvL\nb0VXJ8K/Ls13b3nxlngTTa8cA6868OsP0VkKpjL3iww9Ie23Q/z2osEfXvqa4aa8\nMIqUyiEvj2dOIiMwVkUFRQ/Nr1PTLAAGsiC4lN6w1GVDSLmn574DFVXOCtvIOz21\nXLPikFNzj6Hzo90L/T4cr/i/gH5y6z4MtcQLzv+aa2AKLsEE0gaGAQn01Spp/BB6\nIY7dAO9Uwe1wlaTJHh02tS3yBkygqigzTQ9x7h/9K93q0oRr03M3htiKH57mdViQ\nW6+nZFNjrooLlJdmLidT3Vo2zjyPJPcBFPGz+RPOKOjNe+icB44YbdGOc5p8D1MA\nVpRFTku/WBhhACctRD3YTyyJRPQCc2fEOH2Thm5lCvGe5QzQ0vLBjKU8r464jgGa\nhbLbMOuQaS0uk0aTh7ZZ\n=6PRy\n-----END PGP SIGNATURE-----\n", + "payload": "tree 79ed9f0356648c142701f10ea04f2dcf9b87385d\nparent addf44b05b6dfe2bfbbbb6e6a0753ca9ff6195d3\nauthor Alistair Sellar <16133375+alistairsellar@users.noreply.github.com> 1747326398 +0200\ncommitter GitHub 1747326398 +0200\n\nRemove `test_recipe` command (#4031)\n\n", + "verified_at": "2025-05-15T16:31:42Z", + }, + }, + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/4515a2b9260ddef6b996a1d1b911e5d3d2029638", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/4515a2b9260ddef6b996a1d1b911e5d3d2029638", + "comments_url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/4515a2b9260ddef6b996a1d1b911e5d3d2029638/comments", + "author": { + "login": "alistairsellar", + "id": 16133375, + "node_id": "MDQ6VXNlcjE2MTMzMzc1", + "avatar_url": "https://avatars.githubusercontent.com/u/16133375?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/alistairsellar", + "html_url": "https://github.com/alistairsellar", + "followers_url": "https://api.github.com/users/alistairsellar/followers", + "following_url": "https://api.github.com/users/alistairsellar/following{/other_user}", + "gists_url": "https://api.github.com/users/alistairsellar/gists{/gist_id}", + "starred_url": "https://api.github.com/users/alistairsellar/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/alistairsellar/subscriptions", + "organizations_url": "https://api.github.com/users/alistairsellar/orgs", + "repos_url": "https://api.github.com/users/alistairsellar/repos", + "events_url": "https://api.github.com/users/alistairsellar/events{/privacy}", + "received_events_url": "https://api.github.com/users/alistairsellar/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "committer": { + "login": "web-flow", + "id": 19864447, + "node_id": "MDQ6VXNlcjE5ODY0NDQ3", + "avatar_url": "https://avatars.githubusercontent.com/u/19864447?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/web-flow", + "html_url": "https://github.com/web-flow", + "followers_url": "https://api.github.com/users/web-flow/followers", + "following_url": "https://api.github.com/users/web-flow/following{/other_user}", + "gists_url": "https://api.github.com/users/web-flow/gists{/gist_id}", + "starred_url": "https://api.github.com/users/web-flow/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/web-flow/subscriptions", + "organizations_url": "https://api.github.com/users/web-flow/orgs", + "repos_url": "https://api.github.com/users/web-flow/repos", + "events_url": "https://api.github.com/users/web-flow/events{/privacy}", + "received_events_url": "https://api.github.com/users/web-flow/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": False, + }, + "parents": [ + { + "sha": "addf44b05b6dfe2bfbbbb6e6a0753ca9ff6195d3", + "url": "https://api.github.com/repos/ESMValGroup/ESMValTool/commits/addf44b05b6dfe2bfbbbb6e6a0753ca9ff6195d3", + "html_url": "https://github.com/ESMValGroup/ESMValTool/commit/addf44b05b6dfe2bfbbbb6e6a0753ca9ff6195d3", + } + ], + }, +] diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py index ecdffc056a..7d0fe48989 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_fetch_commit_info.py @@ -32,23 +32,29 @@ ], ids=["todays_shas_only", "todays_shas_same_as_yesterdays"], ) +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.process_commit_info" +) @patch( "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.fetch_single_commit" ) def test_fetch_commit_details_from_github_api_single_commits( - mock_fetch_single, mock_shas_by_package_and_day + mock_fetch_single, mock_process_commit_info, mock_shas_by_package_and_day ): """Test correct function is called for fetching single commits.""" - mock_fetch_single.return_value = {"mock_response": "data"} + mock_fetch_single.return_value = {"mock_raw_commit": "raw_data"} mock_headers = "headers" + mock_process_commit_info.return_value = [ + {"mock_processed_commit": "processed_data"} + ] actual = fetch_commit_details_from_github_api( mock_shas_by_package_and_day, mock_headers ) assert actual == { - "ESMValCore": [{"mock_response": "data"}], - "ESMValTool": [{"mock_response": "data"}], + "ESMValCore": [{"mock_processed_commit": "processed_data"}], + "ESMValTool": [{"mock_processed_commit": "processed_data"}], } assert mock_fetch_single.call_count == 2 @@ -57,12 +63,21 @@ def test_fetch_commit_details_from_github_api_single_commits( call("ESMValTool", "ESMValGroup", mock_headers, "ijkl789"), ] + assert mock_process_commit_info.call_count == 2 + assert mock_process_commit_info.call_args_list == [ + call({"mock_raw_commit": "raw_data"}), + call({"mock_raw_commit": "raw_data"}), + ] + +@patch( + "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.process_commit_info" +) @patch( "esmvaltool.utils.recipe_test_workflow.app.generate_report.bin.fetch_commit_info.fetch_range_of_commits" ) def test_fetch_commit_details_from_github_api_range_of_commits( - mock_fetch_range, + mock_fetch_range, mock_process_commit_info ): """Test correct function is called for fetching a range of commits.""" mock_fetch_range.return_value = [{"mock_response": "data"}] @@ -71,14 +86,17 @@ def test_fetch_commit_details_from_github_api_range_of_commits( "ESMValCore": {"today": "abcd123", "yesterday": "efgh456"}, "ESMValTool": {"today": "ijkl789", "yesterday": "mnop012"}, } + mock_process_commit_info.return_value = [ + {"mock_processed_commit": "processed_data"} + ] actual = fetch_commit_details_from_github_api( mock_shas_by_package_and_day, mock_headers ) assert actual == { - "ESMValCore": [{"mock_response": "data"}], - "ESMValTool": [{"mock_response": "data"}], + "ESMValCore": [{"mock_processed_commit": "processed_data"}], + "ESMValTool": [{"mock_processed_commit": "processed_data"}], } assert mock_fetch_range.call_count == 2 @@ -107,6 +125,7 @@ def test_fetch_single_commit(mock_get): """Test fetching a single commit from the GitHub API.""" mock_response = Mock() mock_response.json.return_value = {"mock_response": "data"} + mock_response.status_code = 200 mock_get.return_value = mock_response actual = fetch_single_commit( @@ -116,6 +135,8 @@ def test_fetch_single_commit(mock_get): mock_get.assert_called_once_with( "https://api.github.com/repos/mock_owner/mock_repo/commits/mock_sha", headers="mock_headers", + params=None, + timeout=10, ) @@ -133,6 +154,7 @@ def test_fetch_range_of_commits_results_on_first_page(mock_get): ] mock_response = Mock() mock_response.json.return_value = mock_response_json_return_value + mock_response.status_code = 200 mock_get.return_value = mock_response mock_params = { @@ -149,9 +171,10 @@ def test_fetch_range_of_commits_results_on_first_page(mock_get): ) assert actual == mock_response_json_return_value[:-1] mock_get.assert_called_once_with( - "https://api.github.com/repos/mock_owner/mock_repo/commits/", + "https://api.github.com/repos/mock_owner/mock_repo/commits", headers="mock_headers", params=mock_params, + timeout=10, ) @@ -173,6 +196,7 @@ def test_fetch_range_of_commits_results_on_later_page(mock_get): mock_response_json_return_value[10:20], # Second page mock_response_json_return_value[20:30], # Third page ] + mock_response.status_code = 200 mock_get.return_value = mock_response actual = fetch_range_of_commits( @@ -193,9 +217,9 @@ def test_fetch_range_of_commits_results_on_later_page(mock_get): # "page": 3, # } # expected_calls = [ - # call(expected_url, headers="mock_headers", params=expected_params), - # call(expected_url, headers="mock_headers", params=expected_params), - # call(expected_url, headers="mock_headers", params=expected_params), + # call(expected_url, headers="mock_headers", params=expected_params, timeout=10), + # call(expected_url, headers="mock_headers", params=expected_params, timeout=10), + # call(expected_url, headers="mock_headers", params=expected_params, timeout=10), # ] # mock_get.assert_has_calls(expected_calls) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py index e8d0a74b30..6ac307489a 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_generate_html_report.py @@ -27,6 +27,9 @@ MockDbData = namedtuple("MockDbdata", ["cycle", "row_data"]) +# TODO: Generate and add. +real_commit_info_for_testing = "" + @pytest.fixture def mock_db_data_single_cycle(): From 55c5d81628fadd392a89ac205ab8fbc0daeef0d6 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Fri, 20 Jun 2025 11:26:40 +0200 Subject: [PATCH 33/40] #4036: Tweak container regex. Working on dkrz. --- .../app/generate_report/bin/shas_via_singularity.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py index 1bbe0f2cf3..a882ee0361 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/shas_via_singularity.py @@ -21,9 +21,8 @@ def get_shas_from_singularity(dev_versions_today, dev_versions_yesterday): Notes ----- - The regex used expects short SHAs between "+g" and a new line in the SCM - version strings. A branch with uncommited changes would break this pattern - but that should not be possible from a container. + The regex used expects short SHAs between "+g" and a "." or new line in + the SCM version string. More on setuptools scm's versioning scheme here under "Default verisioning scheme: @@ -48,7 +47,7 @@ def get_shas_from_singularity(dev_versions_today, dev_versions_yesterday): print("Only today's SHAs are available.") for day, package_versions in pkg_versions_by_day: - shas = re.findall(r"\+g(.*?)$", package_versions, re.MULTILINE) + shas = re.findall(r"\+g(.*?)(?:\.|$)", package_versions, re.MULTILINE) if len(shas) == 2: all_shas["ESMValCore"][day] = shas[0] all_shas["ESMValTool"][day] = shas[1] From 0a1dd53d06cca64ed5d516ac7a25d760ba4477bf Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Wed, 25 Jun 2025 16:05:16 +0100 Subject: [PATCH 34/40] #4036: Add debug logs to report. Working on MO --- .../bin/generate_html_report.py | 173 +++++++++++++++--- .../generate_report/bin/report_template.jinja | 23 ++- 2 files changed, 161 insertions(+), 35 deletions(-) diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index 3b37a80714..acfdce301e 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -3,6 +3,7 @@ import os import sqlite3 +import subprocess import traceback from datetime import datetime from pathlib import Path @@ -30,10 +31,22 @@ # Load environment variables required at all sites. CYLC_DB_PATH = os.environ.get("CYLC_DB_PATH") +CYLC_SHARE_DIR = os.environ.get("CYLC_WORKFLOW_SHARE_DIR") CYLC_TASK_CYCLE_POINT = os.environ.get("CYLC_TASK_CYCLE_POINT") CYLC_TASK_CYCLE_YESTERDAY = os.environ.get("CYLC_TASK_CYCLE_YESTERDAY") +CYLC_WORKFLOW_RUN_DIR = os.environ.get("CYLC_WORKFLOW_RUN_DIR") +OUTPUT_DIR = os.environ.get("OUTPUT_DIR") +PRODUCTION = os.environ.get("PRODUCTION") REPORT_PATH = os.environ.get("REPORT_PATH") SITE = os.environ.get("SITE") +# TODO: Move to main/DKRZ after development. PRODUCTION too? +VM_PATH = os.environ.get("VM_PATH") + +# TODO: Remove after development. +VM_PATH = Path(CYLC_WORKFLOW_RUN_DIR) / "mock_vm" +MOCK_PRODUCTION = True +if VM_PATH: + VM_DEBUG_LOG_DIR = Path(VM_PATH) / "debug_logs" ESMVAL_VERSIONS_TODAY = None ESMVAL_VERSIONS_YESTERDAY = None @@ -87,6 +100,9 @@ def main( A dictionary of git repos if the site uses git repos, or None. """ commit_info = None + raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) + processed_db_data = process_db_output(raw_db_data) + # Commits/SHAs will only be included for these sites. The report will run # at other sites without commit/SHA information. try: @@ -117,8 +133,10 @@ def main( commit_info = fetch_commit_details_from_github_api(sha_info) add_report_messages_to_commits(commit_info) - raw_db_data = fetch_report_data(db_file_path, cylc_task_cycle_point) - processed_db_data = process_db_output(raw_db_data) + # TODO: move to dkrz section after development. + if VM_DEBUG_LOG_DIR and MOCK_PRODUCTION: + debug_log_processor(processed_db_data) + subheader = create_subheader(cylc_task_cycle_point) rendered_html = render_html_report( @@ -274,39 +292,134 @@ def process_db_output(report_data): return sorted_processed_db_data -def copy_debug_log_to_vm(failed_task): - """ """ - # TODO: Current work is in python-playground/lumberjack.py - pass +def copy_a_debug_log_to_vm(target_debug_log, recipe, task): + """ + Copy a debug log file to a VM directory ``debug_logs//``. + Parameters + ---------- + target_debug_log : Path + The target log file. E.g. From the Cylc run directory. + recipe : str + The name of the recipe. + task : str + The name of the task e.g. "process_task", "compare_task". -def add_debug_log_for_failed_tasks(processed_db_data): + Returns + ------- + Path + The path to the debug log on the VM. """ - { - "recipe_1": { - "process_task": { - "status": "succeeded", - "style": "color: green", - "debug_log": "path/to/debug.log", - }, - "compare_task": { - "status": "failed", - "style": "color: red", - "debug_log": "path/to/debug.log", - }, - } - }, + if not VM_DEBUG_LOG_DIR or not MOCK_PRODUCTION: + return None + file_name = target_debug_log.name + vm_debug_log_dir_for_recipe_task = VM_DEBUG_LOG_DIR / recipe / task + + if not vm_debug_log_dir_for_recipe_task.exists(): + vm_debug_log_dir_for_recipe_task.mkdir(parents=True, exist_ok=True) + + command = f"rsync -a {target_debug_log} {vm_debug_log_dir_for_recipe_task}" + subprocess.run(command, shell=True) + return vm_debug_log_dir_for_recipe_task / file_name + + +def esmvaltool_debug_log_processor(recipe): """ + Copy the ESMValTool ``main_log_debug.txt`` to the VM, if it exists. - for recipe, tasks in processed_db_data.items(): - for task_name, task_data in tasks.items(): - if task_data["status"] == "failed": - # Assuming the debug log path is constructed from the recipe name - # and task name. Adjust as necessary. - debug_log_path = ( - f"/path/to/debug/logs/{recipe}/{task_name}.log" - ) - task_data["debug_log"] = debug_log_path + Parameters + ---------- + recipe: str + The recipe. + + Returns + ------- + Path | None + The path to the debug log file on the VM, or None. + """ + if OUTPUT_DIR: + output_dir = Path(OUTPUT_DIR) + if output_dir.is_dir(): + for path in output_dir.iterdir(): + # Recipes in directories need to be split. + recipe_name = recipe.split("/")[-1] + if path.name.startswith(recipe_name): + debug_file_path = path / "run" / "main_log_debug.txt" + if debug_file_path.exists(): + vm_debug_file_path = copy_a_debug_log_to_vm( + debug_file_path, recipe, "process_task" + ) + return vm_debug_file_path + return None + + +def cylc_debug_log_processor(recipe, task): + """ + Copy the Cylc stderr and stdout log files to the VM, if they exist. + + Parameters + ---------- + recipe : str + The name of the recipe. + task : str + The name of the task e.g. "process_task", "compare_task". + + Returns + ------- + dict + Dict of debug logs. The key is the HTML display name (e.g. ``stderr``) + and the value is the path to the debug log on the VM. If no Cylc debug + logs exist, the returned dict will be empty. + """ + # Recipes in directories need to be recombined. + recipe_name = recipe.replace("/", "--") + cylc_debug_logs = {} + task_prefix = task.split("_")[0] + + for display_name, file in [("stderr", "job.err"), ("stdout", "job.out")]: + path_to_run_cylc_log = ( + Path(CYLC_WORKFLOW_RUN_DIR) + / "log" + / "job" + / CYLC_TASK_CYCLE_POINT + / f"{task_prefix}_{recipe_name}" + / "01" + / file + ) + if path_to_run_cylc_log.exists(): + vm_debug_file_path = copy_a_debug_log_to_vm( + path_to_run_cylc_log, recipe, task + ) + cylc_debug_logs[display_name] = vm_debug_file_path + return cylc_debug_logs + + +def debug_log_processor(processed_db_data): + """ + Copy debug logs to the VM and add the VM paths to the database task data. + + Parameters + ---------- + processed_db_data : dict + A dictionary with recipe names as keys and tasks/task data as values. + Debug logs are added to the dict as task data e.g. + ``{"" : "" {"status": ... "debug_logs" { ...}}}`` + """ + for target_task in ("process_task", "compare_task"): + for recipe, task_data in processed_db_data.items(): + target_task_data = task_data.get(target_task) + if target_task_data: # TODO: + if target_task_data.get("status") is not None: # == "failed": + target_task_data["debug_logs"] = {} + cylc_debug_log_paths = cylc_debug_log_processor( + recipe, target_task + ) + target_task_data["debug_logs"].update(cylc_debug_log_paths) + # Only process tasks have ESMValTool debug logs. + if target_task == "process_task": + target_task_data["debug_logs"]["debug"] = ( + esmvaltool_debug_log_processor(recipe) + ) def create_subheader(cylc_task_cycle_point): diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja index 5ba701b26d..f6f726c62e 100644 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/report_template.jinja @@ -48,7 +48,7 @@

{{ subheader }}

{% if esmval_core_commits %}

ESMValTool

TimeSHAAuthorCommit message
{{ commit['report_flag'] }}{{ commit['date'] }}{{ commit['sha'] }}{{ commit['author'] }} + {{ commit['sha'] }} + + Author Avatar + {{ commit['author'] }} + {{ commit['message']}}
- +

ESMValCore:

@@ -80,7 +80,7 @@ {% endif %} {% if esmval_tool_commits %}

ESMValCore
Time
- +

ESMValTool:

@@ -114,7 +114,8 @@

ESMValTool
Time
- +

Test Results:

+

For failed tasks, links to the debug log, stdout and stderr are added if they exist.

@@ -123,8 +124,20 @@ {%- for recipe, tasks in report_data.items() -%} - - + {%- set process_task = tasks.get("process_task", {}) -%} + + {%- set compare_task = tasks.get("compare_task", {}) -%} + {%- endfor -%}

Test Results
Recipe Recipe Run
{{ recipe }}{{ tasks.get('process_task', {}).get('status', '-') }}{{ tasks.get('compare_task', {}).get('status', '-') }} + {{ process_task.get('status', '-') }} + {% for display_name, path in process_task.get("debug_logs", {}).items() %} + ({{ display_name }}) + {% endfor %} + + {{ compare_task.get('status', '-') }} + {% for display_name, path in compare_task.get("debug_logs", {}).items() %} + ({{ display_name }}) + {% endfor %} +
From 5c046d499688623a9c1b4527e29dc73960bd00e7 Mon Sep 17 00:00:00 2001 From: Chris Billows Date: Thu, 26 Jun 2025 14:48:25 +0100 Subject: [PATCH 35/40] #4036: Fix backslashes. Handle missing esmv debugs. Add report warning --- .../bin/generate_html_report.py | 69 +- .../generate_report/bin/report_template.jinja | 3 +- .../app/generate_report/bin/test_data.py | 742 ------------------ setup.cfg | 6 + 4 files changed, 57 insertions(+), 763 deletions(-) delete mode 100644 esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/test_data.py create mode 100644 setup.cfg diff --git a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py index acfdce301e..288d69734e 100755 --- a/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py +++ b/esmvaltool/utils/recipe_test_workflow/app/generate_report/bin/generate_html_report.py @@ -2,6 +2,7 @@ """Generate a HTML summary report from a Cylc SQLite database.""" import os +import shutil import sqlite3 import subprocess import traceback @@ -39,14 +40,16 @@ PRODUCTION = os.environ.get("PRODUCTION") REPORT_PATH = os.environ.get("REPORT_PATH") SITE = os.environ.get("SITE") -# TODO: Move to main/DKRZ after development. PRODUCTION too? -VM_PATH = os.environ.get("VM_PATH") -# TODO: Remove after development. +# TODO: Remove/move to main/DKRZ after development. PRODUCTION too? +VM_PATH = os.environ.get("VM_PATH") VM_PATH = Path(CYLC_WORKFLOW_RUN_DIR) / "mock_vm" MOCK_PRODUCTION = True if VM_PATH: VM_DEBUG_LOG_DIR = Path(VM_PATH) / "debug_logs" + if VM_DEBUG_LOG_DIR.exists(): + shutil.rmtree(VM_DEBUG_LOG_DIR) + VM_DEBUG_LOG_DIR.mkdir(parents=True, exist_ok=True) ESMVAL_VERSIONS_TODAY = None ESMVAL_VERSIONS_YESTERDAY = None @@ -56,7 +59,6 @@ ESMVAL_VERSIONS_TODAY = os.environ.get("ESMVAL_VERSIONS_CURRENT") ESMVAL_VERSIONS_YESTERDAY = os.environ.get("ESMVAL_VERSIONS_PREVIOUS") - if SITE == "metoffice": REPOS = { "ESMValCore_today": os.environ.get("ESMVALCORE_DIR"), @@ -133,10 +135,11 @@ def main( commit_info = fetch_commit_details_from_github_api(sha_info) add_report_messages_to_commits(commit_info) - # TODO: move to dkrz section after development. + # TODO: move to dkrz section after development and use PRODUCTION. if VM_DEBUG_LOG_DIR and MOCK_PRODUCTION: debug_log_processor(processed_db_data) + reinstate_backslashes_to_recipe_names(processed_db_data) subheader = create_subheader(cylc_task_cycle_point) rendered_html = render_html_report( @@ -237,7 +240,7 @@ def process_db_task(task_name, status): recipe_name = task_name_parts[1] processed_task_name = task_name_parts[0] + "_task" # Restore directories to a "/" - recipe_name = task_name_parts[1].replace("--", "/") + # recipe_name = task_name_parts[1].replace("--", "/") style = styles.get(status, "color: black") task_data = ( recipe_name, @@ -334,23 +337,26 @@ def esmvaltool_debug_log_processor(recipe): Returns ------- - Path | None - The path to the debug log file on the VM, or None. + dict + A dict containing the path to the debug log file on the VM, or an empty + dict. """ if OUTPUT_DIR: output_dir = Path(OUTPUT_DIR) if output_dir.is_dir(): + # ESMValTool only uses the last part of a recipe name when creating + # it's directory. E.g. ``examples--recipe_python`` will be in + # ``recipe_python__