diff --git a/README.md b/README.md index 0d4df95..b7f42d3 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,9 @@ chromium output/linuxserver/plex/latest/index.html | `WEB_AUTH` | Credentials for basic auth, format `user:password`. Leave empty for none. | `""` | | `WEB_SCREENSHOT_DELAY` | Seconds to wait after the page loads before taking the screenshot. | `20` | +### Generating a package diff + +In development mode you can build an actual comparison image by tagging it as `testimage:latest`. This will simulate the pacakge difference diff by dumping the SBOM of your test image and comparing it to the last release of the repo. ## Advanced Usage (CI Environment) diff --git a/ci/ci.py b/ci/ci.py index 6d9f607..43b6593 100755 --- a/ci/ci.py +++ b/ci/ci.py @@ -11,6 +11,8 @@ import mimetypes import json import subprocess +import io +from PIL import Image from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from functools import wraps @@ -125,6 +127,8 @@ def __init__(self) -> None: self.region: str = os.environ.get("S3_REGION", "us-east-1") self.bucket: str = os.environ.get("S3_BUCKET", "ci-tests.linuxserver.io") self.release_tag: str = os.environ.get("RELEASE_TAG", "latest") + self.release_type: str = os.environ.get("RELEASE_TYPE", "stable") + self.ls_branch: str = os.environ.get("LS_BRANCH", "master") self.syft_image_tag: str = os.environ.get("SYFT_IMAGE_TAG", "v1.26.1") self.commit_sha: str = os.environ.get("COMMIT_SHA", "") self.build_number: str = os.environ.get("BUILD_NUMBER", "") @@ -159,6 +163,7 @@ def __init__(self) -> None: BASE: '{os.environ.get("BASE")}' META_TAG: '{os.environ.get("META_TAG")}' RELEASE_TAG: '{os.environ.get("RELEASE_TAG")}' + RELEASE_TYPE: '{os.environ.get("RELEASE_TYPE")}' TAGS: '{os.environ.get("TAGS")}' S6_VERBOSITY: '{os.environ.get("S6_VERBOSITY")}' CI_S6_VERBOSITY '{os.environ.get("CI_S6_VERBOSITY")}' @@ -386,18 +391,21 @@ def container_test(self, tag: str) -> None: self._endtest(container, tag, build_info, sbom, False, start_time) return + # Calculate package diff + package_diff = self.get_package_diff(sbom) + # Screenshot the web interface and check connectivity screenshot_success, browser_logs = self.take_screenshot(container, tag) if not screenshot_success and self.get_platform(tag) == Platform.AMD64.value: self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time) - self._endtest(container, tag, build_info, sbom, False, start_time, browser_logs) + self._endtest(container, tag, build_info, sbom, False, start_time, browser_logs, package_diff) return - self._endtest(container, tag, build_info, sbom, True, start_time, browser_logs) + self._endtest(container, tag, build_info, sbom, True, start_time, browser_logs, package_diff) self.logger.success("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time) return - def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str|CITestResult, test_success: bool, start_time:float|int = 0.0, browser_logs: str = "") -> None: + def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str|CITestResult, test_success: bool, start_time:float|int = 0.0, browser_logs: str = "", package_diff: str = "") -> None: """End the test with as much info as we have and append to the report. Args: @@ -408,6 +416,7 @@ def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packa `test_success` (bool): If the testing of the container failed or not `start_time` (float, optional): The start time of the test. Defaults to 0.0. Used to calculate the runtime of the test. `browser_logs` (str, optional): The browser console logs. + `package_diff` (str, optional): The diff of packages between this build and the last release. """ if not start_time: runtime = "-" @@ -429,6 +438,7 @@ def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packa self.report_containers[tag] = { "logs": logblob, "sysinfo": packages, + "package_diff": package_diff, "browser_logs": browser_logs, "warnings": { "dotnet": warning_texts["dotnet"] if "icu-libs" in packages and "arm32" in tag else "", @@ -596,6 +606,13 @@ def make_sbom(self, tag: str) -> "str|CITestResult": self.logger.warning("Falling back to Syft for SBOM generation on tag %s", tag) # Fallback to syft if buildx failed + if os.environ.get("CI_LOCAL_MODE", "false").lower() == "true": + self.logger.info("Local mode detected, attempting to generate SBOM from local 'testimage:latest'") + sbom = self.get_sbom_syft(tag, override_image="testimage:latest") + if sbom != CITestResult.ERROR: + self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.PASS, "Generated from testimage:latest", start_time) + self.create_html_ansi_file(str(sbom),tag,"sbom") + return sbom sbom = self.get_sbom_syft(tag) if sbom != CITestResult.ERROR: self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.PASS, "-", start_time) @@ -605,23 +622,34 @@ def make_sbom(self, tag: str) -> "str|CITestResult": self.report_status = CIReportResult.FAIL self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.FAIL, "Failed to generate SBOM with both buildx and syft", start_time) return CITestResult.ERROR - - def get_sbom_syft(self, tag: str) -> str | CITestResult: + + def get_sbom_syft(self, tag: str, override_image: str = None) -> str | CITestResult: """Get the SBOM for the image tag using Syft. Args: tag (str): The tag we are testing + override_image (str, optional): Use this image name instead of self.image:tag. Defaults to None. Returns: str: SBOM output if successful, otherwise "ERROR". """ start_time = time.time() platform: str = self.get_platform(tag) - syft:Container = self.client.containers.run(image=f"ghcr.io/anchore/syft:{self.syft_image_tag}",command=f"{self.image}:{tag} --platform=linux/{platform}", + target_image = override_image if override_image else f"{self.image}:{tag}" + try: + self.client.images.get(target_image) + except ImageNotFound: + self.logger.error("Image %s not found, cannot generate Syft SBOM", target_image) + return CITestResult.ERROR + except APIError as error: + self.logger.error("API error while checking for image %s: %s", target_image, error) + return CITestResult.ERROR + cmd = f"{target_image}" if override_image else f"{target_image} --platform=linux/{platform}" + syft:Container = self.client.containers.run(image=f"ghcr.io/anchore/syft:{self.syft_image_tag}",command=cmd, detach=True, volumes={"/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"}}) - self.logger.info("Creating SBOM package list on %s with syft version %s",tag,self.syft_image_tag) + self.logger.info("Creating SBOM package list on %s with syft version %s", target_image, self.syft_image_tag) logblob: str = "" t_end: float = time.time() + self.sbom_timeout - self.logger.info("Tailing the Syft container logs for %s seconds looking the 'VERSION' message on tag: %s",self.sbom_timeout,tag) + self.logger.info("Tailing the Syft container logs for %s seconds looking the 'VERSION' message on tag: %s",self.sbom_timeout,tag) while time.time() < t_end: time.sleep(5) try: @@ -1047,6 +1075,74 @@ def _add_test_result(self, tag:str, test:CITests, status:CITestResult, message:s "message":message, "runtime": runtime}.items()))) + def get_package_diff(self, current_sbom: str | CITestResult) -> str: + """Fetch the last release/branch SBOM and generate a diff against the current SBOM.""" + if isinstance(current_sbom, CITestResult): + return "" + try: + # Determine repo name + container_name = self.image.split("/")[-1] + for prefix in ["lspipepr-", "lsiodev-"]: + if container_name.startswith(prefix): + container_name = container_name.replace(prefix, "") + if self.release_type == "stable": + repo_api = f"https://api.github.com/repos/linuxserver/docker-{container_name}/releases/latest" + resp = requests.get(repo_api, timeout=10) + if resp.status_code == 404: + self.logger.info("No release found for %s, skipping package diff", container_name) + return "" + if resp.status_code != 200: + self.logger.warning("Could not fetch latest release info from GitHub: %s", resp.status_code) + return "" + tag_name = resp.json().get("tag_name") + if not tag_name: + return "" + raw_sbom_url = f"https://raw.githubusercontent.com/linuxserver/docker-{container_name}/refs/tags/{tag_name}/package_versions.txt" + else: + raw_sbom_url = f"https://raw.githubusercontent.com/linuxserver/docker-{container_name}/refs/heads/{self.ls_branch}/package_versions.txt" + # Get remote SBOM + resp_sbom = requests.get(raw_sbom_url, timeout=10) + if resp_sbom.status_code == 404: + self.logger.info("No package_versions.txt found at %s, skipping package diff", raw_sbom_url) + return "" + if resp_sbom.status_code != 200: + self.logger.warning("Could not fetch remote SBOM from %s: %s", raw_sbom_url, resp_sbom.status_code) + return "" + remote_pkgs = self._parse_sbom_string(resp_sbom.text) + current_pkgs = self._parse_sbom_string(current_sbom) + return self._generate_diff_text(remote_pkgs, current_pkgs) + except Exception: + self.logger.exception("Failed to generate package diff") + return "" + + def _parse_sbom_string(self, sbom_text: str) -> dict[str, str]: + """Parse the formatted SBOM table into a dictionary.""" + pkgs = {} + lines = sbom_text.strip().splitlines() + # Skip header if present + if lines and "NAME" in lines[0] and "VERSION" in lines[0]: + lines = lines[1:] + for line in lines: + parts = line.split() + if len(parts) >= 2: + pkgs[parts[0]] = parts[1] + return pkgs + + def _generate_diff_text(self, old_pkgs: dict[str, str], new_pkgs: dict[str, str]) -> str: + """Generate a text diff between two package lists.""" + diff_lines = [] + all_keys = set(old_pkgs.keys()) | set(new_pkgs.keys()) + for pkg in sorted(all_keys): + old_ver = old_pkgs.get(pkg) + new_ver = new_pkgs.get(pkg) + if old_ver is None: + diff_lines.append(f"[+] {pkg}: {new_ver} (Added)") + elif new_ver is None: + diff_lines.append(f"[-] {pkg}: {old_ver} (Removed)") + elif old_ver != new_ver: + diff_lines.append(f"[*] {pkg}: {old_ver} -> {new_ver} (Changed)") + return "\n".join(diff_lines) if diff_lines else "No package changes found." + def take_screenshot(self, container: Container, tag:str) -> tuple[bool, str]: """Take a screenshot and save it to self.outdir if self.screenshot is True @@ -1080,9 +1176,14 @@ def take_screenshot(self, container: Container, tag:str) -> tuple[bool, str]: driver.get(endpoint) time.sleep(self.screenshot_delay) # A grace period for the page to load self.logger.debug("Trying to take screenshot of %s at %s", tag, endpoint) - driver.get_screenshot_as_file(f"{self.outdir}/{tag}.png") - if not os.path.isfile(f"{self.outdir}/{tag}.png"): - raise FileNotFoundError(f"Screenshot '{self.outdir}/{tag}.png' not found") + + png_data = driver.get_screenshot_as_png() + image = Image.open(io.BytesIO(png_data)) + rgb_im = image.convert('RGB') + rgb_im.save(f"{self.outdir}/{tag}.jpg", quality=80) + + if not os.path.isfile(f"{self.outdir}/{tag}.jpg"): + raise FileNotFoundError(f"Screenshot '{self.outdir}/{tag}.jpg' not found") self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.PASS, "-", start_time) self.logger.success("Screenshot %s: PASSED after %.2f seconds", tag, time.time() - start_time) return True, self._get_browser_logs(driver, tag) diff --git a/ci/template.html b/ci/template.html index 86b1957..fdf73b3 100644 --- a/ci/template.html +++ b/ci/template.html @@ -1,5 +1,4 @@ - @@ -30,528 +29,605 @@ - + + + + + + + - @@ -559,94 +635,184 @@
+

LinuxServer.io

+
-

Test Results

-

{{ image }}

-

{{ meta_tag }}

-

Cumulative: {{ report_status }}

- Total Runtime: {{ total_runtime }} +
+

CI Test Results

+ +
+ docker pull {{ image }}:{{ meta_tag }} + + + +
+ + + +
+ Cumulative Status: + {% if report_status.lower() == 'pass' %} + PASS + {% else %} + FAIL + {% endif %} +
+
Total Runtime: {{ total_runtime }}
+
+
{% for tag in report_containers %}
-
- {% if report_containers[tag]["test_success"] %} -

{{ report_containers[tag]["platform"] }} PASS

- {% else %} -

{{ report_containers[tag]["platform"] }} FAIL

- {% endif %} -

- {% if report_status.lower() == "pass" %} - {{ image }} + +
+
+

+ {% if report_status.lower() == "pass" %} + {{ image }} + {% else %} + {{ image }}:{{ tag }} + {% endif %} +

+
Runtime: {{ report_containers[tag]["runtime"] }}
+
+
+ {{ report_containers[tag]["platform"] }} + {% if report_containers[tag]["test_success"] %} + PASS {% else %} - {{ image }}:{{ tag }} + FAIL {% endif %} -

-
Runtime: {{ report_containers[tag]["runtime"] }}
- {% if screenshot %} - - {{ tag }} - - {% else %} -
- WEB_SCREENSHOT ENV Disabled +
- {% endif %} -
Build Information
-
+ +
+ {% if screenshot %} + + Screenshot for {{ tag }} + + {% else %} +
+ + WEB_SCREENSHOT ENV Disabled +
+ {% endif %} +
+ +
{% for key, value in report_containers[tag]["build_info"].items() %} -
- {{ key|capitalize }}: {{ value }} +
+ {{ key }} + {{ value }}
{% endfor %}
- - View Container Logs - -
- Expand -
- 
{{ report_containers[tag]["logs"] }}
-
-
- - View SBOM output - -
- Expand -
- 
{{ report_containers[tag]["sysinfo"] }}
-
-
- {% if report_containers[tag]["browser_logs"] %} - - View Browser Console Logs - -
- Expand -
- 
{{ report_containers[tag]["browser_logs"] }}
-
-
- {% endif %} - {% if report_containers[tag]["has_warnings"]%} -
- Warnings - {% for warning in report_containers[tag]["warnings"] %} - {% if report_containers[tag]["warnings"][warning] %} -
- {{ report_containers[tag]["warnings"][warning] }} + +
+ + +
+ +
+ Container Logs
- {% endif %} - {% endfor %} -
- {% endif %} + +
+ + 
{{ report_containers[tag]["logs"] }}
+
+
+ + +
+ +
+ SBOM Output +
+ +
+ + 
{{ report_containers[tag]["sysinfo"] }}
+
+
+ + + {% if report_containers[tag]["package_diff"] %} +
+ +
+ Package Changes +
+ +
+ 
{{ report_containers[tag]["package_diff"] }}
+
+
+ {% endif %} + + + {% if report_containers[tag]["browser_logs"] %} +
+ +
+ Browser Console Logs +
+ +
+ + 
{{ report_containers[tag]["browser_logs"] }}
+
+
+ {% endif %} + + + {% if report_containers[tag]["has_warnings"] %} +
+ +
+ Warnings Detected +
+ +
+ {% for warning in report_containers[tag]["warnings"] %} + {% if report_containers[tag]["warnings"][warning] %} +
+

{{ report_containers[tag]["warnings"][warning] }}

+
+ {% endif %} + {% endfor %} +
+
+ {% endif %} + +
+ +
- + @@ -656,46 +822,63 @@

{% for test in report_containers[tag]["test_results"] %} - + + {% if report_containers[tag]["test_results"][test]['status'] == 'PASS' %} - + {% else %} - + {% endif %} - - + + + {% endfor %}
Test Result Message
{{ test }}{{ test }}{{ report_containers[tag]["test_results"][test]['status'] }} {{ report_containers[tag]["test_results"][test]['status'] }} {{ report_containers[tag]["test_results"][test]['status'] }} {{ report_containers[tag]["test_results"][test]['status'] }} {{ report_containers[tag]["test_results"][test]["message"] }}{{ report_containers[tag]["test_results"][test]["runtime"] }}{{ report_containers[tag]["test_results"][test]["message"] }}{{ report_containers[tag]["test_results"][test]["runtime"] }}
+ {% endfor %}
+ +
- - View Python Logs -
- Expand - 

+        
+          

+             View Python CI Logs
+          

+        
+        

+          

+             Open Full Python Log
+          

+          
 Loading logs...

+        

       

     

+
   

+
   
 
-
 
diff --git a/readme-vars.yml b/readme-vars.yml
index 01bead7..839ea0e 100644
--- a/readme-vars.yml
+++ b/readme-vars.yml
@@ -82,6 +82,9 @@ full_custom_readme: |
   | `WEB_AUTH` | Credentials for basic auth, format `user:password`. Leave empty for none. | `""` |
   | `WEB_SCREENSHOT_DELAY` | Seconds to wait after the page loads before taking the screenshot. | `20` |
 
+  ### Generating a package diff
+  
+  In development mode you can build an actual comparison image by tagging it as `testimage:latest`. This will simulate the pacakge difference diff by dumping the SBOM of your test image and comparing it to the last release of the repo.
 
   ## Advanced Usage (CI Environment)
 
diff --git a/requirements.txt b/requirements.txt
index 4764ccd..82c4981 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ jinja2==3.1.2
 requests==2.28.2
 pyvirtualdisplay==3.0
 ansi2html==1.8.0
+pillow==12.1.1
diff --git a/tests/test_ci.py b/tests/test_ci.py
index 98dba30..69a2ec0 100644
--- a/tests/test_ci.py
+++ b/tests/test_ci.py
@@ -150,7 +150,7 @@ def test_watch_container_logs(ci: CI, mock_container: Mock):
 def test_take_screenshot(ci:CI,mock_container: Mock):
     screenshot: bool = ci.take_screenshot(mock_container, ci.tags[0])
     if screenshot:
-        assert os.path.isfile(os.path.join(ci.outdir, f"{ci.tags[0]}.png")) is True
+        assert os.path.isfile(os.path.join(ci.outdir, f"{ci.tags[0]}.jpg")) is True
         assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CAPTURE_SCREENSHOT.value]["status"] == CITestResult.PASS.value
     else:
         assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CAPTURE_SCREENSHOT.value]["status"] == CITestResult.FAIL.value