From 98b7833e380ccd10ca705dcbfe2d946a43a267d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Sep 2025 10:52:48 +0000 Subject: [PATCH 1/4] Initial plan From e49f19c3208eb43085cc6b8adeae00eba1981e35 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:00:04 +0000 Subject: [PATCH 2/4] Add Palace container support and update CI workflow for docs Co-authored-by: nikosavola <7860886+nikosavola@users.noreply.github.com> --- .github/workflows/pages.yml | 40 ++++++++++++ gplugins/palace/__init__.py | 3 + gplugins/palace/get_capacitance.py | 45 ++++++-------- gplugins/palace/get_scattering.py | 7 ++- gplugins/palace/utils.py | 99 ++++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 27 deletions(-) create mode 100644 gplugins/palace/utils.py diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 292ce289..a3fb9ef4 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -16,9 +16,49 @@ jobs: steps: - uses: actions/checkout@v5 - uses: astral-sh/setup-uv@v6 + - name: Setup Apptainer + uses: eWaterCycle/setup-apptainer@3f706d898c9db585b1d741b4692e66755f3a1b40 + with: + apptainer-version: 1.4.2 - name: Install libglu1-mesa run: | sudo apt-get install -y libglu1-mesa + - name: Download Palace singularity definition + run: wget https://raw.githubusercontent.com/awslabs/palace/main/singularity/singularity.def + - name: Cache Palace container restore + id: cache-palace-restore + uses: actions/cache/restore@v4 + with: + path: palace.sif + key: palace-container-${{ runner.os }}-${{ hashFiles('singularity.def') }} + - name: Build palace from source + if: steps.cache-palace-restore.outputs.cache-hit != 'true' + run: | + echo "Building Palace container from source (this may take a while)…" + echo "Note: This is a one-time build that will be cached for future runs" + timeout 3600 sudo apptainer build palace.sif singularity.def || { + echo "Build timed out or failed after 60 minutes. Palace will not be available for docs." + exit 0 + } + - name: Cache Palace container save + uses: actions/cache/save@v4 + if: steps.cache-palace-restore.outputs.cache-hit != 'true' + with: + path: palace.sif + key: palace-container-${{ runner.os }}-${{ hashFiles('singularity.def') }} + - name: Create palace alias + if: hashFiles('palace.sif') != '' + run: | + # Create a shell script that acts as an alias for palace + echo '#!/bin/bash' > palace + echo "exec apptainer run $GITHUB_WORKSPACE/palace.sif \"\$@\" " >> palace + chmod +x palace + echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH + echo "Palace executable created successfully" + - name: Verify Palace installation + if: hashFiles('palace.sif') != '' + run: | + palace --help || echo "Palace help failed but container exists" - name: Build docs env: SIMCLOUD_APIKEY: ${{ secrets.SIMCLOUD_APIKEY }} diff --git a/gplugins/palace/__init__.py b/gplugins/palace/__init__.py index 4e139213..c26fa3af 100644 --- a/gplugins/palace/__init__.py +++ b/gplugins/palace/__init__.py @@ -1,7 +1,10 @@ from gplugins.palace.get_capacitance import run_capacitive_simulation_palace from gplugins.palace.get_scattering import run_scattering_simulation_palace +from gplugins.palace.utils import find_palace_executable, run_palace_command __all__ = [ "run_capacitive_simulation_palace", "run_scattering_simulation_palace", + "find_palace_executable", + "run_palace_command", ] diff --git a/gplugins/palace/get_capacitance.py b/gplugins/palace/get_capacitance.py index ce4689d9..d1c2c950 100644 --- a/gplugins/palace/get_capacitance.py +++ b/gplugins/palace/get_capacitance.py @@ -153,24 +153,26 @@ def _generate_json( def _palace(simulation_folder: Path, name: str, n_processes: int = 1) -> None: """Run simulations with Palace.""" - # Try to find palace in PATH first - palace = shutil.which("palace") + from gplugins.palace.utils import find_palace_executable - # If not found, try to load it via Spack + json_file = simulation_folder / f"{Path(name).stem}.json" + + print(f"🔍 DEBUG: Running Palace simulation...") + print(f" JSON config file: {json_file}") + print(f" Simulation folder: {simulation_folder}") + print(f" Working directory contents before Palace:") + for item in simulation_folder.iterdir(): + print(f" - {item.name}") + + # Try to find palace executable (PATH, containers, etc.) + palace = find_palace_executable() + if palace is None: - print(" Palace not found in PATH, attempting to load via Spack...") - # Create a command that sources Spack and then runs palace - json_file = simulation_folder / f"{Path(name).stem}.json" + # Fallback to Spack method + print(" Palace not found in PATH or containers, attempting to load via Spack...") spack_cmd = f"source {home}/install_new_computer/bash/spack/share/spack/setup-env.fish && spack load palace && palace {json_file.absolute()}" - print(f"🔍 DEBUG: Running Palace simulation via Spack...") print(f" Command: {spack_cmd}") - print(f" JSON config file: {json_file}") - print(f" Simulation folder: {simulation_folder}") - print(f" Working directory contents before Palace:") - for item in simulation_folder.iterdir(): - print(f" - {item.name}") - try: import subprocess @@ -197,27 +199,20 @@ def _palace(simulation_folder: Path, name: str, n_processes: int = 1) -> None: except Exception as e: print(f" ❌ Failed to run Palace via Spack: {e}") raise RuntimeError( - "palace not found. Make sure it is available in your PATH or via Spack." + "palace not found. Make sure it is available in your PATH, " + "via Spack, or via an Apptainer/Singularity container." ) else: - # Palace found in PATH, use the original method - json_file = simulation_folder / f"{Path(name).stem}.json" - - print(f"🔍 DEBUG: Running Palace simulation...") + # Palace found, use async execution method print(f" Palace executable: {palace}") - print(f" JSON config file: {json_file}") - print(f" Simulation folder: {simulation_folder}") - print(f" Working directory contents before Palace:") - for item in simulation_folder.iterdir(): - print(f" - {item.name}") try: run_async_with_event_loop( execute_and_stream_output( ( - [palace, json_file] + [palace, str(json_file)] if n_processes == 1 - else [palace, "-np", str(n_processes), json_file] + else [palace, "-np", str(n_processes), str(json_file)] ), shell=False, log_file_dir=simulation_folder, diff --git a/gplugins/palace/get_scattering.py b/gplugins/palace/get_scattering.py index efe51c9a..67de7a03 100644 --- a/gplugins/palace/get_scattering.py +++ b/gplugins/palace/get_scattering.py @@ -212,6 +212,8 @@ async def _palace( simulation_folder: Path, json_files: Collection[Path], n_processes: int = 1 ) -> None: """Run simulations with Palace.""" + from gplugins.palace.utils import find_palace_executable + # split processes as evenly as possible quotient, remainder = divmod(n_processes, len(json_files)) n_processes_per_json = [quotient] * len(json_files) @@ -220,10 +222,11 @@ async def _palace( n_processes_per_json[i] + 1, 1 ) # need at least one - palace = shutil.which("palace") + palace = find_palace_executable() if palace is None: raise RuntimeError( - "`palace` not found. Make sure it is available in your PATH." + "`palace` not found. Make sure it is available in your PATH, " + "via Spack, or via an Apptainer/Singularity container." ) tasks = [ diff --git a/gplugins/palace/utils.py b/gplugins/palace/utils.py new file mode 100644 index 00000000..06dc311d --- /dev/null +++ b/gplugins/palace/utils.py @@ -0,0 +1,99 @@ +"""Utilities for Palace execution.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path +from typing import Any + + +def find_palace_executable() -> str | None: + """Find Palace executable in various locations. + + Returns: + Path to palace executable, or None if not found. + """ + # First try to find palace in PATH + palace = shutil.which("palace") + if palace: + return palace + + # Check if we're in a CI environment with Apptainer container + if "GITHUB_WORKSPACE" in os.environ: + # Look for palace.sif in GITHUB_WORKSPACE + workspace = Path(os.environ["GITHUB_WORKSPACE"]) + palace_sif = workspace / "palace.sif" + if palace_sif.exists(): + # Create a temporary script to run palace via apptainer + palace_script = workspace / "palace_runner.sh" + palace_script.write_text( + f'#!/bin/bash\nexec apptainer run "{palace_sif}" "$@"\n' + ) + palace_script.chmod(0o755) + return str(palace_script) + + # Check common container locations + for sif_path in [ + Path.home() / "palace.sif", + Path("/opt/palace.sif"), + Path("/app/palace.sif"), + ]: + if sif_path.exists(): + # Create a temporary script to run palace via apptainer/singularity + script_dir = Path.home() / ".local" / "bin" + script_dir.mkdir(parents=True, exist_ok=True) + palace_script = script_dir / "palace_apptainer" + + # Try apptainer first, then singularity + for container_cmd in ["apptainer", "singularity"]: + if shutil.which(container_cmd): + palace_script.write_text( + f'#!/bin/bash\nexec {container_cmd} run "{sif_path}" "$@"\n' + ) + palace_script.chmod(0o755) + return str(palace_script) + + return None + + +def run_palace_command( + command: list[str], + cwd: Path | str | None = None, + capture_output: bool = True, + text: bool = True, + **kwargs: Any, +) -> subprocess.CompletedProcess[str]: + """Run a palace command, automatically detecting the correct executable. + + Args: + command: Command list where the first element should be "palace" + cwd: Working directory + capture_output: Whether to capture stdout/stderr + text: Whether to use text mode + **kwargs: Additional arguments to subprocess.run + + Returns: + subprocess.CompletedProcess result + + Raises: + RuntimeError: If palace is not found or execution fails + """ + palace_exe = find_palace_executable() + if palace_exe is None: + raise RuntimeError( + "palace not found. Make sure it is available in your PATH, " + "via Spack, or via an Apptainer/Singularity container." + ) + + # Replace "palace" with the actual executable path + full_command = [palace_exe] + command[1:] + + return subprocess.run( + full_command, + cwd=cwd, + capture_output=capture_output, + text=text, + **kwargs, + ) \ No newline at end of file From ec7ed85195db176936183de975ae80e82eab44cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:06:18 +0000 Subject: [PATCH 3/4] Improve Palace container detection and CI workflow robustness Co-authored-by: nikosavola <7860886+nikosavola@users.noreply.github.com> --- .github/workflows/pages.yml | 26 ++++++++++++++++---------- gplugins/palace/utils.py | 27 +++++++++++++++------------ 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index a3fb9ef4..2436700e 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -42,23 +42,29 @@ jobs: } - name: Cache Palace container save uses: actions/cache/save@v4 - if: steps.cache-palace-restore.outputs.cache-hit != 'true' + if: steps.cache-palace-restore.outputs.cache-hit != 'true' && hashFiles('palace.sif') != '' with: path: palace.sif key: palace-container-${{ runner.os }}-${{ hashFiles('singularity.def') }} - name: Create palace alias - if: hashFiles('palace.sif') != '' run: | - # Create a shell script that acts as an alias for palace - echo '#!/bin/bash' > palace - echo "exec apptainer run $GITHUB_WORKSPACE/palace.sif \"\$@\" " >> palace - chmod +x palace - echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH - echo "Palace executable created successfully" + if [ -f "palace.sif" ]; then + # Create a shell script that acts as an alias for palace + echo '#!/bin/bash' > palace + echo "exec apptainer run $GITHUB_WORKSPACE/palace.sif \"\$@\" " >> palace + chmod +x palace + echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH + echo "Palace executable created successfully" + else + echo "Palace container not available - notebooks requiring Palace will be skipped" + fi - name: Verify Palace installation - if: hashFiles('palace.sif') != '' run: | - palace --help || echo "Palace help failed but container exists" + if [ -f "palace.sif" ]; then + palace --help || echo "Palace help failed but container exists" + else + echo "Palace container not found, skipping verification" + fi - name: Build docs env: SIMCLOUD_APIKEY: ${{ secrets.SIMCLOUD_APIKEY }} diff --git a/gplugins/palace/utils.py b/gplugins/palace/utils.py index 06dc311d..226ee967 100644 --- a/gplugins/palace/utils.py +++ b/gplugins/palace/utils.py @@ -26,13 +26,16 @@ def find_palace_executable() -> str | None: workspace = Path(os.environ["GITHUB_WORKSPACE"]) palace_sif = workspace / "palace.sif" if palace_sif.exists(): - # Create a temporary script to run palace via apptainer - palace_script = workspace / "palace_runner.sh" - palace_script.write_text( - f'#!/bin/bash\nexec apptainer run "{palace_sif}" "$@"\n' - ) - palace_script.chmod(0o755) - return str(palace_script) + # Check if apptainer/singularity is available + for container_cmd in ["apptainer", "singularity"]: + if shutil.which(container_cmd): + # Create a temporary script to run palace via container + palace_script = workspace / "palace_runner.sh" + palace_script.write_text( + f'#!/bin/bash\nexec {container_cmd} run "{palace_sif}" "$@"\n' + ) + palace_script.chmod(0o755) + return str(palace_script) # Check common container locations for sif_path in [ @@ -41,14 +44,14 @@ def find_palace_executable() -> str | None: Path("/app/palace.sif"), ]: if sif_path.exists(): - # Create a temporary script to run palace via apptainer/singularity - script_dir = Path.home() / ".local" / "bin" - script_dir.mkdir(parents=True, exist_ok=True) - palace_script = script_dir / "palace_apptainer" - # Try apptainer first, then singularity for container_cmd in ["apptainer", "singularity"]: if shutil.which(container_cmd): + # Create a temporary script to run palace via container + script_dir = Path.home() / ".local" / "bin" + script_dir.mkdir(parents=True, exist_ok=True) + palace_script = script_dir / "palace_apptainer" + palace_script.write_text( f'#!/bin/bash\nexec {container_cmd} run "{sif_path}" "$@"\n' ) From 7adcc11558578cfabed43ead7699faf265e72793 Mon Sep 17 00:00:00 2001 From: Niko Savola Date: Thu, 18 Sep 2025 20:24:46 +0300 Subject: [PATCH 4/4] Revert changes done to Palace code by Copilot --- gplugins/palace/__init__.py | 3 - gplugins/palace/get_capacitance.py | 45 +++++++------ gplugins/palace/get_scattering.py | 7 +- gplugins/palace/utils.py | 102 ----------------------------- 4 files changed, 27 insertions(+), 130 deletions(-) delete mode 100644 gplugins/palace/utils.py diff --git a/gplugins/palace/__init__.py b/gplugins/palace/__init__.py index c26fa3af..4e139213 100644 --- a/gplugins/palace/__init__.py +++ b/gplugins/palace/__init__.py @@ -1,10 +1,7 @@ from gplugins.palace.get_capacitance import run_capacitive_simulation_palace from gplugins.palace.get_scattering import run_scattering_simulation_palace -from gplugins.palace.utils import find_palace_executable, run_palace_command __all__ = [ "run_capacitive_simulation_palace", "run_scattering_simulation_palace", - "find_palace_executable", - "run_palace_command", ] diff --git a/gplugins/palace/get_capacitance.py b/gplugins/palace/get_capacitance.py index d1c2c950..ce4689d9 100644 --- a/gplugins/palace/get_capacitance.py +++ b/gplugins/palace/get_capacitance.py @@ -153,26 +153,24 @@ def _generate_json( def _palace(simulation_folder: Path, name: str, n_processes: int = 1) -> None: """Run simulations with Palace.""" - from gplugins.palace.utils import find_palace_executable + # Try to find palace in PATH first + palace = shutil.which("palace") - json_file = simulation_folder / f"{Path(name).stem}.json" - - print(f"🔍 DEBUG: Running Palace simulation...") - print(f" JSON config file: {json_file}") - print(f" Simulation folder: {simulation_folder}") - print(f" Working directory contents before Palace:") - for item in simulation_folder.iterdir(): - print(f" - {item.name}") - - # Try to find palace executable (PATH, containers, etc.) - palace = find_palace_executable() - + # If not found, try to load it via Spack if palace is None: - # Fallback to Spack method - print(" Palace not found in PATH or containers, attempting to load via Spack...") + print(" Palace not found in PATH, attempting to load via Spack...") + # Create a command that sources Spack and then runs palace + json_file = simulation_folder / f"{Path(name).stem}.json" spack_cmd = f"source {home}/install_new_computer/bash/spack/share/spack/setup-env.fish && spack load palace && palace {json_file.absolute()}" + print(f"🔍 DEBUG: Running Palace simulation via Spack...") print(f" Command: {spack_cmd}") + print(f" JSON config file: {json_file}") + print(f" Simulation folder: {simulation_folder}") + print(f" Working directory contents before Palace:") + for item in simulation_folder.iterdir(): + print(f" - {item.name}") + try: import subprocess @@ -199,20 +197,27 @@ def _palace(simulation_folder: Path, name: str, n_processes: int = 1) -> None: except Exception as e: print(f" ❌ Failed to run Palace via Spack: {e}") raise RuntimeError( - "palace not found. Make sure it is available in your PATH, " - "via Spack, or via an Apptainer/Singularity container." + "palace not found. Make sure it is available in your PATH or via Spack." ) else: - # Palace found, use async execution method + # Palace found in PATH, use the original method + json_file = simulation_folder / f"{Path(name).stem}.json" + + print(f"🔍 DEBUG: Running Palace simulation...") print(f" Palace executable: {palace}") + print(f" JSON config file: {json_file}") + print(f" Simulation folder: {simulation_folder}") + print(f" Working directory contents before Palace:") + for item in simulation_folder.iterdir(): + print(f" - {item.name}") try: run_async_with_event_loop( execute_and_stream_output( ( - [palace, str(json_file)] + [palace, json_file] if n_processes == 1 - else [palace, "-np", str(n_processes), str(json_file)] + else [palace, "-np", str(n_processes), json_file] ), shell=False, log_file_dir=simulation_folder, diff --git a/gplugins/palace/get_scattering.py b/gplugins/palace/get_scattering.py index 67de7a03..efe51c9a 100644 --- a/gplugins/palace/get_scattering.py +++ b/gplugins/palace/get_scattering.py @@ -212,8 +212,6 @@ async def _palace( simulation_folder: Path, json_files: Collection[Path], n_processes: int = 1 ) -> None: """Run simulations with Palace.""" - from gplugins.palace.utils import find_palace_executable - # split processes as evenly as possible quotient, remainder = divmod(n_processes, len(json_files)) n_processes_per_json = [quotient] * len(json_files) @@ -222,11 +220,10 @@ async def _palace( n_processes_per_json[i] + 1, 1 ) # need at least one - palace = find_palace_executable() + palace = shutil.which("palace") if palace is None: raise RuntimeError( - "`palace` not found. Make sure it is available in your PATH, " - "via Spack, or via an Apptainer/Singularity container." + "`palace` not found. Make sure it is available in your PATH." ) tasks = [ diff --git a/gplugins/palace/utils.py b/gplugins/palace/utils.py deleted file mode 100644 index 226ee967..00000000 --- a/gplugins/palace/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Utilities for Palace execution.""" - -from __future__ import annotations - -import os -import shutil -import subprocess -from pathlib import Path -from typing import Any - - -def find_palace_executable() -> str | None: - """Find Palace executable in various locations. - - Returns: - Path to palace executable, or None if not found. - """ - # First try to find palace in PATH - palace = shutil.which("palace") - if palace: - return palace - - # Check if we're in a CI environment with Apptainer container - if "GITHUB_WORKSPACE" in os.environ: - # Look for palace.sif in GITHUB_WORKSPACE - workspace = Path(os.environ["GITHUB_WORKSPACE"]) - palace_sif = workspace / "palace.sif" - if palace_sif.exists(): - # Check if apptainer/singularity is available - for container_cmd in ["apptainer", "singularity"]: - if shutil.which(container_cmd): - # Create a temporary script to run palace via container - palace_script = workspace / "palace_runner.sh" - palace_script.write_text( - f'#!/bin/bash\nexec {container_cmd} run "{palace_sif}" "$@"\n' - ) - palace_script.chmod(0o755) - return str(palace_script) - - # Check common container locations - for sif_path in [ - Path.home() / "palace.sif", - Path("/opt/palace.sif"), - Path("/app/palace.sif"), - ]: - if sif_path.exists(): - # Try apptainer first, then singularity - for container_cmd in ["apptainer", "singularity"]: - if shutil.which(container_cmd): - # Create a temporary script to run palace via container - script_dir = Path.home() / ".local" / "bin" - script_dir.mkdir(parents=True, exist_ok=True) - palace_script = script_dir / "palace_apptainer" - - palace_script.write_text( - f'#!/bin/bash\nexec {container_cmd} run "{sif_path}" "$@"\n' - ) - palace_script.chmod(0o755) - return str(palace_script) - - return None - - -def run_palace_command( - command: list[str], - cwd: Path | str | None = None, - capture_output: bool = True, - text: bool = True, - **kwargs: Any, -) -> subprocess.CompletedProcess[str]: - """Run a palace command, automatically detecting the correct executable. - - Args: - command: Command list where the first element should be "palace" - cwd: Working directory - capture_output: Whether to capture stdout/stderr - text: Whether to use text mode - **kwargs: Additional arguments to subprocess.run - - Returns: - subprocess.CompletedProcess result - - Raises: - RuntimeError: If palace is not found or execution fails - """ - palace_exe = find_palace_executable() - if palace_exe is None: - raise RuntimeError( - "palace not found. Make sure it is available in your PATH, " - "via Spack, or via an Apptainer/Singularity container." - ) - - # Replace "palace" with the actual executable path - full_command = [palace_exe] + command[1:] - - return subprocess.run( - full_command, - cwd=cwd, - capture_output=capture_output, - text=text, - **kwargs, - ) \ No newline at end of file