Add jinja templating, payload as tar, tests

.github/workflows/main.yml

@@ -121,7 +121,9 @@ jobs:
             conda create -yqp "${{ runner.temp }}/conda-standalone-nightly" -c conda-canary/label/dev "conda-standalone=*=*single*"
             echo "CONSTRUCTOR_CONDA_EXE=${{ runner.temp }}/conda-standalone-nightly/standalone_conda/conda.exe" >> $GITHUB_ENV
           elif [[ "${{ matrix.conda-standalone }}" == "conda-standalone-onedir" ]]; then
-            conda create -yqp "${{ runner.temp }}/conda-standalone-onedir" -c conda-canary/label/dev "conda-standalone=*=*onedir*"
+            # Request a version newer than 25.1.1 due to an issue with newer versions getting deprioritized 
+            # because they are built with 'track_features'
+            conda create -yqp "${{ runner.temp }}/conda-standalone-onedir" -c conda-canary/label/dev "conda-standalone>25.1.1=*onedir*"
             echo "CONSTRUCTOR_CONDA_EXE=${{ runner.temp }}/conda-standalone-onedir/standalone_conda/conda.exe" >> $GITHUB_ENV
           else
             conda activate constructor-dev
@@ -153,7 +155,7 @@ jobs:
           AZURE_SIGNTOOL_KEY_VAULT_URL: ${{ secrets.AZURE_SIGNTOOL_KEY_VAULT_URL }}
           CONSTRUCTOR_EXAMPLES_KEEP_ARTIFACTS: "${{ runner.temp }}/examples_artifacts"
           CONSTRUCTOR_SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.26100.0/x86/signtool.exe"
-          CONSTRUCTOR_VERBOSE: 1
+          CONSTRUCTOR_VERBOSE: 0
         run: |
           rm -rf coverage.json
           pytest -vv --cov=constructor --cov-branch tests/test_examples.py

constructor/briefcase.py

@@ -2,11 +2,14 @@
 Logic to build installers using Briefcase.
 """
 
+import functools
 import logging
+import os
 import re
 import shutil
 import sys
 import sysconfig
+import tarfile
 import tempfile
 from dataclasses import dataclass
 from pathlib import Path
@@ -19,6 +22,7 @@
     tomli_w = None  # This file is only intended for Windows use
 
 from . import preconda
+from .jinja import render_template
 from .utils import DEFAULT_REVERSE_DOMAIN_ID, copy_conda_exe, filename_dist
 
 BRIEFCASE_DIR = Path(__file__).parent / "briefcase"
@@ -219,37 +223,6 @@ def create_install_options_list(info: dict) -> list[dict]:
     return options
 
 
-# Create a Briefcase configuration file. Using a full TOML writer rather than a Jinja
-# template allows us to avoid escaping strings everywhere.
-def write_pyproject_toml(tmp_dir, info):
-    name, version = get_name_version(info)
-    bundle, app_name = get_bundle_app_name(info, name)
-
-    config = {
-        "project_name": name,
-        "bundle": bundle,
-        "version": version,
-        "license": get_license(info),
-        "app": {
-            app_name: {
-                "formal_name": f"{info['name']} {info['version']}",
-                "description": "",  # Required, but not used in the installer.
-                "external_package_path": EXTERNAL_PACKAGE_PATH,
-                "use_full_install_path": False,
-                "install_launcher": False,
-                "post_install_script": str(BRIEFCASE_DIR / "run_installation.bat"),
-                "install_option": create_install_options_list(info),
-            }
-        },
-    }
-
-    if "company" in info:
-        config["author"] = info["company"]
-
-    (tmp_dir / "pyproject.toml").write_text(tomli_w.dumps({"tool": {"briefcase": config}}))
-    logger.debug(f"Created TOML file at: {tmp_dir}")
-
-
 @dataclass(frozen=True)
 class PayloadLayout:
     """A data class with purpose to contain the payload layout."""
@@ -267,29 +240,135 @@ class Payload:
     """
 
     info: dict
-    root: Path | None = None
+    archive_name: str = "payload.tar.gz"
+    conda_exe_name: str = "_conda.exe"
+
+    # Enable additional log output during pre/post uninstall/install.
+    add_debug_logging: bool = False
+
+    @functools.cached_property
+    def root(self) -> Path:
+        """Create root upon first access and cache it."""
+        return Path(tempfile.mkdtemp(prefix="payload-"))
+
+    def remove(self, *, ignore_errors: bool = True) -> None:
+        """Remove the root of the payload.
+
+        This function requires some extra care due to the root being a cached property.
+        """
+        root = getattr(self, "root", None)
+        if root is None:
+            return
+        shutil.rmtree(root, ignore_errors=ignore_errors)
+        # Now we drop the cached value so next access will recreate if desired
+        try:
+            delattr(self, "root")
+        except Exception:
+            # delattr on a cached_property may raise on some versions / edge cases
+            pass
 
     def prepare(self) -> PayloadLayout:
-        root = self._ensure_root()
-        self._write_pyproject(root)
+        """Prepares the payload."""
+        root = self.root
         layout = self._create_layout(root)
+        # Render the template files and add them to the necessary config field
+        self.render_templates()
+        self.write_pyproject_toml(layout)
 
         preconda.write_files(self.info, layout.base)
         preconda.copy_extra_files(self.info.get("extra_files", []), layout.external)
         self._stage_dists(layout)
         self._stage_conda(layout)
+
+        archive_path = self.make_archive(layout.base, layout.external)
+        if not archive_path.exists():
+            raise RuntimeError(f"Unexpected error, failed to create archive: {archive_path}")
         return layout
 
-    def remove(self) -> None:
-        shutil.rmtree(self.root)
+    def make_archive(self, src: Path, dst: Path) -> Path:
+        """Create an archive of the directory 'src'.
+        The input 'src' must be an existing directory.
+        If 'dst' does not exist, this function will create it.
+        The directory specified via 'src' is removed after successful creation.
+        Returns the path to the archive.
+
+        Example:
+            payload = Payload(...)
+            foo = Path('foo')
+            bar = Path('bar')
+            targz = payload.make_archive(foo, bar)
+            This will create the file bar\\<payload.archive_name> containing 'foo' and all its contents.
+
+        """
+        if not src.is_dir():
+            raise NotADirectoryError(src)
+        dst.mkdir(parents=True, exist_ok=True)
+
+        archive_path = dst / self.archive_name
+
+        archive_type = archive_path.suffix[1:]  # since suffix starts with '.'
+        with tarfile.open(archive_path, mode=f"w:{archive_type}", compresslevel=1) as tar:
+            tar.add(src, arcname=src.name)
+
+        shutil.rmtree(src)
+        return archive_path
+
+    def render_templates(self) -> list[Path]:
+        """Render the configured templates under the payload root,
+        returns a list of Paths to the rendered templates.
+        """
+        templates = {
+            Path(BRIEFCASE_DIR / "run_installation.bat"): Path(self.root / "run_installation.bat"),
+            Path(BRIEFCASE_DIR / "pre_uninstall.bat"): Path(self.root / "pre_uninstall.bat"),
+        }
+
+        context: dict[str, str] = {
+            "archive_name": self.archive_name,
+            "conda_exe_name": self.conda_exe_name,
+            "add_debug": self.add_debug_logging,
+            "register_envs": str(self.info.get("register_envs", True)).lower(),
+        }
+
+        # Render the templates now using jinja and the defined context
+        for src, dst in templates.items():
+            if not src.exists():
+                raise FileNotFoundError(src)
+            rendered = render_template(src.read_text(encoding="utf-8"), **context)
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            dst.write_text(rendered, encoding="utf-8", newline="\r\n")
+
+        return list(templates.values())
+
+    def write_pyproject_toml(self, layout: PayloadLayout) -> None:
+        name, version = get_name_version(self.info)
+        bundle, app_name = get_bundle_app_name(self.info, name)
+
+        config = {
+            "project_name": name,
+            "bundle": bundle,
+            "version": version,
+            "license": get_license(self.info),
+            "app": {
+                app_name: {
+                    "formal_name": f"{self.info['name']} {self.info['version']}",
+                    "description": "",  # Required, but not used in the installer.
+                    "external_package_path": str(layout.external),
+                    "use_full_install_path": False,
+                    "install_launcher": False,
+                    "install_option": create_install_options_list(self.info),
+                    "post_install_script": str(layout.root / "run_installation.bat"),
+                    "pre_uninstall_script": str(layout.root / "pre_uninstall.bat"),
+                }
+            },
+        }
 
-    def _write_pyproject(self, root: Path) -> None:
-        write_pyproject_toml(root, self.info)
+        # Add optional content
+        if "company" in self.info:
+            config["author"] = self.info["company"]
 
-    def _ensure_root(self) -> Path:
-        if self.root is None:
-            self.root = Path(tempfile.mkdtemp())
-        return self.root
+        # Finalize
+        (layout.root / "pyproject.toml").write_text(tomli_w.dumps({"tool": {"briefcase": config}}))
+        logger.debug(f"Created TOML file at: {layout.root}")
 
     def _create_layout(self, root: Path) -> PayloadLayout:
         """The layout is created as:
@@ -315,7 +394,7 @@ def _stage_dists(self, layout: PayloadLayout) -> None:
             shutil.copy(download_dir / filename_dist(dist), layout.pkgs)
 
     def _stage_conda(self, layout: PayloadLayout) -> None:
-        copy_conda_exe(layout.external, "_conda.exe", self.info["_conda_exe"])
+        copy_conda_exe(layout.external, self.conda_exe_name, self.info["_conda_exe"])
 
 
 def create(info, verbose=False):

constructor/briefcase/pre_uninstall.bat

@@ -0,0 +1,53 @@
+@echo {{ 'on' if add_debug else 'off' }}
+setlocal
+
+{% macro error_block(message, code) %}
+echo [ERROR] {{ message }}
+>> "%LOG%" echo [ERROR] {{ message }}
+exit /b {{ code }}
+{% endmacro %}
+
+rem Assign INSTDIR and normalize the path
+set "INSTDIR=%~dp0.."
+for %%I in ("%INSTDIR%") do set "INSTDIR=%%~fI"
+
+set "BASE_PATH=%INSTDIR%\base"
+set "PREFIX=%BASE_PATH%"
+set "CONDA_EXE=%INSTDIR%\{{ conda_exe_name }}"
+set "PAYLOAD_TAR=%INSTDIR%\{{ archive_name }}"
+
+rem Get the name of the install directory
+for %%I in ("%INSTDIR%") do set "APPNAME=%%~nxI"
+set "LOG=%INSTDIR%\uninstall.log"
+
+{%- if add_debug %}
+echo ==== pre_uninstall start ==== >> "%LOG%"
+echo SCRIPT=%~f0 >> "%LOG%"
+echo CWD=%CD% >> "%LOG%"
+echo INSTDIR=%INSTDIR% >> "%LOG%"
+echo BASE_PATH=%BASE_PATH% >> "%LOG%"
+echo CONDA_EXE=%CONDA_EXE% >> "%LOG%"
+echo PAYLOAD_TAR=%PAYLOAD_TAR% >> "%LOG%"
+"%CONDA_EXE%" --version >> "%LOG%" 2>&1
+{%- endif %}
+
+rem Consistency checks
+if not exist "%CONDA_EXE%" (
+  {{ error_block('CONDA_EXE not found: "%CONDA_EXE%"', 10) }}
+)
+
+rem Recreate an empty payload tar. This file was deleted during installation but the
+rem MSI installer expects it to exist.
+type nul > "%PAYLOAD_TAR%"
+if errorlevel 1 (
+  {{ error_block('Failed to create "%PAYLOAD_TAR%"', '%errorlevel%') }}
+)
+
+"%CONDA_EXE%" --log-file "%LOG%" constructor uninstall --prefix "%BASE_PATH%"
+if errorlevel 1 ( exit /b %errorlevel% )
+
+rem If we reached this far without any errors, remove any log-files.
+if exist "%INSTDIR%\install.log" del "%INSTDIR%\install.log"
+if exist "%INSTDIR%\uninstall.log" del "%INSTDIR%\uninstall.log"
+
+exit /b 0

constructor/briefcase/run_installation.bat

@@ -1,14 +1,68 @@
-set "INSTDIR=%cd%"
+@echo {{ 'on' if add_debug else 'off' }}
+setlocal
+
+{% macro error_block(message, code) %}
+echo [ERROR] {{ message }}
+>> "%LOG%" echo [ERROR] {{ message }}
+exit /b {{ code }}
+{% endmacro %}
+
+rem Assign INSTDIR and normalize the path
+set "INSTDIR=%~dp0.."
+for %%I in ("%INSTDIR%") do set "INSTDIR=%%~fI"
+
 set "BASE_PATH=%INSTDIR%\base"
 set "PREFIX=%BASE_PATH%"
-set "CONDA_EXE=%INSTDIR%\_conda.exe"
-
-"%INSTDIR%\_conda.exe" constructor --prefix "%BASE_PATH%" --extract-conda-pkgs
+set "CONDA_EXE=%INSTDIR%\{{ conda_exe_name }}"
+set "PAYLOAD_TAR=%INSTDIR%\{{ archive_name }}"
 
+set CONDA_EXTRA_SAFETY_CHECKS=no
 set CONDA_PROTECT_FROZEN_ENVS=0
-set "CONDA_ROOT_PREFIX=%BASE_PATH%"
+set CONDA_REGISTER_ENVS={{ register_envs }}
 set CONDA_SAFETY_CHECKS=disabled
-set CONDA_EXTRA_SAFETY_CHECKS=no
+set "CONDA_ROOT_PREFIX=%BASE_PATH%"
 set "CONDA_PKGS_DIRS=%BASE_PATH%\pkgs"
 
-"%INSTDIR%\_conda.exe" install --offline --file "%BASE_PATH%\conda-meta\initial-state.explicit.txt" -yp "%BASE_PATH%"
+rem Get the name of the install directory
+for %%I in ("%INSTDIR%") do set "APPNAME=%%~nxI"
+set "LOG=%INSTDIR%\install.log"
+
+{%- if add_debug %}
+echo ==== run_installation start ==== >> "%LOG%"
+echo SCRIPT=%~f0 >> "%LOG%"
+echo CWD=%CD% >> "%LOG%"
+echo INSTDIR=%INSTDIR% >> "%LOG%"
+echo BASE_PATH=%BASE_PATH% >> "%LOG%"
+echo CONDA_EXE=%CONDA_EXE% >> "%LOG%"
+echo PAYLOAD_TAR=%PAYLOAD_TAR% >> "%LOG%"
+{%- endif %}
+
+rem Consistency checks
+if not exist "%CONDA_EXE%" (
+  {{ error_block('CONDA_EXE not found: "%CONDA_EXE%"', 10) }}
+)
+if not exist "%PAYLOAD_TAR%" (
+  {{ error_block('PAYLOAD_TAR not found: "%PAYLOAD_TAR%"', 11) }}
+)
+
+echo Unpacking payload...
+"%CONDA_EXE%" --log-file "%LOG%" constructor extract --prefix "%INSTDIR%" --tar-from-stdin < "%PAYLOAD_TAR%"
+if errorlevel 1 ( exit /b %errorlevel% )
+
+"%CONDA_EXE%" --log-file "%LOG%" constructor extract --prefix "%BASE_PATH%" --conda-pkgs
+if errorlevel 1 ( exit /b %errorlevel% )
+
+if not exist "%BASE_PATH%" (
+  {{ error_block('"%BASE_PATH%" not found!', 12) }}
+)
+
+"%CONDA_EXE%" --log-file "%LOG%" install --offline --file "%BASE_PATH%\conda-meta\initial-state.explicit.txt" -yp "%BASE_PATH%"
+if errorlevel 1 ( exit /b %errorlevel% )
+
+rem Delete the payload to save disk space.
+rem A truncated placeholder of 0 bytes is recreated during uninstall
+rem because MSI expects the file to be there to clean the registry.
+del "%PAYLOAD_TAR%"
+if errorlevel 1 ( exit /b %errorlevel% )
+
+exit /b 0

examples/register_envs/construct.yaml

@@ -3,7 +3,7 @@
 
 name: RegisterEnvs
 version: 1.0.0
-installer_type: {{ "exe" if os.name == "nt" else "all" }}
+installer_type: all
 channels:
   - https://repo.anaconda.com/pkgs/main/
 specs: