From 4b110f3f1feed5b872578212dd74a8b681515a29 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Tue, 21 Jan 2025 14:53:03 -0600 Subject: [PATCH 01/39] Adds user-friendly messaging when the adaptor executable isn't found. Also checks the result of `which`, just in case there's an alias that's pointing to nowhere. --- .../process/_logging_subprocess.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index 8376446c..2e4a803b 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging +import shutil import signal import subprocess import uuid @@ -64,7 +65,31 @@ def __init__( # In Windows, this is required for signal. SIGBREAK will be sent to the entire process group. # Without this one, current process will also get the SIGBREAK and may react incorrectly. popen_params.update(creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) # type: ignore[attr-defined] - self._process = subprocess.Popen(**popen_params) + + try: + self._process = subprocess.Popen(**popen_params) + + except FileNotFoundError as fnf_error: + # In ManagedProcess we prepend the executable to the list of arguments before creating a LoggingSubprocess + executable = args[0] + exe_path = shutil.which(executable) + + # If we didn't find the executable found by which + if type(exe_path) != None: + raise FileNotFoundError( + f"Could not find adaptor executable at: {exe_path} using alias {executable}\n" + f"Error:{fnf_error}" + ) + + raise FileNotFoundError( + f"Could not find the executable associated with the adaptor: {executable}\n" + f"Is the executable on the PATH or in the startup directory?\n" + f"Error:{fnf_error}" + ) + + except Exception as error: + raise error + if not self._process.stdout: # pragma: no cover raise RuntimeError("process stdout not set") From 1fee6ec977539a8163f5bd77e5a12446a40b371d Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Tue, 21 Jan 2025 14:53:03 -0600 Subject: [PATCH 02/39] feat! Adds user-friendly messaging when the adaptor executable isn't found Also checks the result of `which`, just in case there's an alias that's pointing to nowhere. Signed-off-by: Justin Blagden --- .../process/_logging_subprocess.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index 8376446c..ef874d7b 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging +import shutil import signal import subprocess import uuid @@ -21,7 +22,11 @@ class LoggingSubprocess(object): - """A process whose stdout/stderr lines are sent to a configurable logger""" + """A process whose stdout/stderr lines are sent to a configurable logger + + Raises: + FileNotFoundError: When the executable the adaptor is set to run cannot be found. + """ _logger: logging.Logger _process: subprocess.Popen @@ -64,7 +69,30 @@ def __init__( # In Windows, this is required for signal. SIGBREAK will be sent to the entire process group. # Without this one, current process will also get the SIGBREAK and may react incorrectly. popen_params.update(creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) # type: ignore[attr-defined] - self._process = subprocess.Popen(**popen_params) + + try: + self._process = subprocess.Popen(**popen_params) + + except FileNotFoundError as fnf_error: + # In ManagedProcess we prepend the executable to the list of arguments before creating a LoggingSubprocess + executable = args[0] + exe_path = shutil.which(executable) + + # If we didn't find the executable found by which + if type(exe_path) is not None: + raise FileNotFoundError( + f"Could not find adaptor executable at: {exe_path} using alias {executable}\n" + f"Error:{fnf_error}" + ) + + raise FileNotFoundError( + f"Could not find the executable associated with the adaptor: {executable}\n" + f"Is the executable on the PATH or in the startup directory?\n" + f"Error:{fnf_error}" + ) + + except Exception as error: + raise error if not self._process.stdout: # pragma: no cover raise RuntimeError("process stdout not set") From 687b3848d0b19c0731b72e673360be2578b34b82 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Tue, 28 Jan 2025 09:22:51 -0600 Subject: [PATCH 03/39] Resolves code smells called out by sonarqube Fixed an always-true type check Removed an explicit throw of un-caught exception types --- src/openjd/adaptor_runtime/process/_logging_subprocess.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index 4fcda528..241dc5b8 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -78,7 +78,7 @@ def __init__( exe_path = shutil.which(executable) # If we didn't find the executable found by which - if type(exe_path) is not None: + if exe_path is not None: raise FileNotFoundError( f"Could not find adaptor executable at: {exe_path} using alias {executable}\n" f"Error:{fnf_error}" @@ -90,9 +90,6 @@ def __init__( f"Error:{fnf_error}" ) - except Exception as error: - raise error - if not self._process.stdout: # pragma: no cover raise RuntimeError("process stdout not set") if not self._process.stderr: # pragma: no cover From 61c67a14513ee3aa5e32f62d58fedecd12fb9bc7 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Mon, 3 Feb 2025 13:25:14 -0600 Subject: [PATCH 04/39] Adds a test for executable not found error --- .../process/test_integration_logging_subprocess.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py b/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py index c524104a..277c33bb 100644 --- a/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py +++ b/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py @@ -113,9 +113,9 @@ def test_startup_directory(self, startup_dir: str | None, caplog): def test_startup_directory_empty_posix(self): """When calling LoggingSubprocess with an empty cwd, FileNotFoundError will be raised.""" args = ["pwd"] - with pytest.raises(FileNotFoundError) as excinfo: + with pytest.raises(FileNotFoundError) as exc_info: LoggingSubprocess(args=args, startup_directory="") - assert "[Errno 2] No such file or directory: ''" in str(excinfo.value) + assert "[Errno 2] No such file or directory: ''" in str(exc_info.value) @pytest.mark.skipif(not OSName.is_windows(), reason="Only run this test in Windows.") def test_startup_directory_empty_windows(self): @@ -151,6 +151,12 @@ def test_log_levels(self, log_level: int, caplog): assert any(r.message == message and r.levelno == _STDERR_LEVEL for r in records) + def test_executable_not_found(self): + """When calling LoggingSubprocess with a missing executable, FileNotFoundError will be raised""" + args = ["missing_executable"] + with pytest.raises(FileNotFoundError) as exc_info: + LoggingSubprocess(args=args) + assert "Could not find the executable associated with the adaptor" in str(exc_info.value) class TestIntegrationRegexHandler(object): """Integration tests for LoggingSubprocess""" From 85bad2976fdc2b95ac0bffe375fda8ecfa0868f4 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Mon, 3 Feb 2025 13:25:14 -0600 Subject: [PATCH 05/39] Adds a test for executable not found error --- .../process/test_integration_logging_subprocess.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py b/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py index c524104a..474ccabb 100644 --- a/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py +++ b/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py @@ -113,9 +113,9 @@ def test_startup_directory(self, startup_dir: str | None, caplog): def test_startup_directory_empty_posix(self): """When calling LoggingSubprocess with an empty cwd, FileNotFoundError will be raised.""" args = ["pwd"] - with pytest.raises(FileNotFoundError) as excinfo: + with pytest.raises(FileNotFoundError) as exc_info: LoggingSubprocess(args=args, startup_directory="") - assert "[Errno 2] No such file or directory: ''" in str(excinfo.value) + assert "[Errno 2] No such file or directory: ''" in str(exc_info.value) @pytest.mark.skipif(not OSName.is_windows(), reason="Only run this test in Windows.") def test_startup_directory_empty_windows(self): @@ -151,6 +151,13 @@ def test_log_levels(self, log_level: int, caplog): assert any(r.message == message and r.levelno == _STDERR_LEVEL for r in records) + def test_executable_not_found(self): + """When calling LoggingSubprocess with a missing executable, FileNotFoundError will be raised""" + args = ["missing_executable"] + with pytest.raises(FileNotFoundError) as exc_info: + LoggingSubprocess(args=args) + assert "Could not find the executable associated with the adaptor" in str(exc_info.value) + class TestIntegrationRegexHandler(object): """Integration tests for LoggingSubprocess""" From 8e59494f831343d58c2ce7e24df43c52a77e83a0 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Fri, 21 Feb 2025 14:13:27 -0600 Subject: [PATCH 06/39] Updates test assertions to include the whole executable path Signed-off-by: Justin Blagden --- src/openjd/adaptor_runtime/process/_logging_subprocess.py | 2 +- .../adaptor_runtime/unit/process/test_logging_subprocess.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index 4b261e7d..ec5a6b37 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -60,7 +60,7 @@ def __init__( if executable_path is None: raise FileNotFoundError( - f"Could not find the executable associated with the adaptor: {executable_path}\n" + f"Could not find the executable associated with the adaptor: {args[0]}\n" f"Is the executable on the PATH or in the startup directory?\n" ) args[0] = executable_path diff --git a/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py b/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py index c79dda95..cc57ad21 100644 --- a/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py +++ b/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py @@ -462,7 +462,10 @@ def test_command_printed(self, mock_popen: mock.Mock, caplog): args = ["cat", "foo.txt"] LoggingSubprocess(args=args) - assert "Running command: /usr/bin/cat foo.txt" in caplog.text + if OSName.is_linux(): + assert "Running command: /usr/bin/cat foo.txt" in caplog.text + elif OSName.is_macos(): + assert "Running command: /bin/cat foo.txt" in caplog.text @mock.patch.object(logging_subprocess.subprocess, "Popen", autospec=True) def test_startup_directory_default(self, mock_popen_autospec: mock.Mock): From 3f320fc1becd223c0f7e065bee705aaeb74a33e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2024 22:18:43 +0000 Subject: [PATCH 07/39] chore(deps): update python-semantic-release requirement (#159) Updates the requirements on [python-semantic-release](https://github.com/python-semantic-release/python-semantic-release) to permit the latest version. - [Release notes](https://github.com/python-semantic-release/python-semantic-release/releases) - [Changelog](https://github.com/python-semantic-release/python-semantic-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/python-semantic-release/python-semantic-release/compare/v9.12...v9.14) --- updated-dependencies: - dependency-name: python-semantic-release dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-release.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-release.txt b/requirements-release.txt index 55f73d99..c49048b3 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1 +1 @@ -python-semantic-release == 9.12.* \ No newline at end of file +python-semantic-release == 9.14.* \ No newline at end of file From 68ad89652d1c1010115c7b156b6c3bf10dfe887c Mon Sep 17 00:00:00 2001 From: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Date: Thu, 28 Nov 2024 16:42:10 -0600 Subject: [PATCH 08/39] chore: allow pywin32 >= 307 (#162) Signed-off-by: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a2691202..2c25f359 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ dependencies = [ "pyyaml ~= 6.0", "jsonschema >= 4.17.0, == 4.*", - "pywin32 == 308; platform_system == 'Windows'", + "pywin32 >= 307; platform_system == 'Windows'", ] [project.urls] From c41feab15ec971efe111d4df63bfe0dddbcd3470 Mon Sep 17 00:00:00 2001 From: client-software-ci <129794699+client-software-ci@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:38:17 -0600 Subject: [PATCH 09/39] chore(release): 0.8.2 (#165) Signed-off-by: client-software-ci <129794699+client-software-ci@users.noreply.github.com> Signed-off-by: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4270dbf9..de9526a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.8.2 (2024-11-29) + +This release allows pywin32 version 307 and above to be used as a dependency for this package. Previously, only pywin32 version 308 was allowed. + + ## 0.8.1 (2024-11-13) From 415ef9f1f173639259698ff305eb1584eeb49bd8 Mon Sep 17 00:00:00 2001 From: Karthik Bekal Pattathana <133984042+karthikbekalp@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:39:20 -0600 Subject: [PATCH 10/39] fix!: Ensure all open uses UTF-8 instead of default encoding. (#171) Signed-off-by: Karthik Bekal Pattathana <133984042+karthikbekalp@users.noreply.github.com> Signed-off-by: Justin Blagden --- .github/scripts/get_latest_changelog.py | 2 +- .../_background/backend_runner.py | 4 +- .../_background/frontend_runner.py | 9 +-- .../adaptor_runtime/_background/loaders.py | 2 +- .../_background/log_buffers.py | 4 +- src/openjd/adaptor_runtime/_entrypoint.py | 8 ++- .../adaptor_runtime/adaptors/_validator.py | 4 +- .../adaptors/configuration/_configuration.py | 6 +- .../configuration/_configuration_manager.py | 2 +- .../integ/_utils/test_secure_open.py | 6 +- .../integ/background/test_background_mode.py | 2 +- .../configuration/test_configuration.py | 69 ++++++++++++++++--- .../test_configuration_manager.py | 2 +- .../unit/background/test_backend_runner.py | 4 +- .../unit/background/test_frontend_runner.py | 2 +- .../unit/background/test_log_buffers.py | 4 +- .../adaptor_runtime/unit/test_entrypoint.py | 4 +- test/openjd/test_copyright_header.py | 2 +- 18 files changed, 97 insertions(+), 39 deletions(-) diff --git a/.github/scripts/get_latest_changelog.py b/.github/scripts/get_latest_changelog.py index 5616efbc..c6a7d33c 100644 --- a/.github/scripts/get_latest_changelog.py +++ b/.github/scripts/get_latest_changelog.py @@ -32,7 +32,7 @@ import re h2 = r"^##\s.*$" -with open("CHANGELOG.md") as f: +with open("CHANGELOG.md", encoding="utf-8") as f: contents = f.read() matches = re.findall(h2, contents, re.MULTILINE) changelog = contents[: contents.find(matches[1]) - 1] if len(matches) > 1 else contents diff --git a/src/openjd/adaptor_runtime/_background/backend_runner.py b/src/openjd/adaptor_runtime/_background/backend_runner.py index 5d4e16e2..a1f7329e 100644 --- a/src/openjd/adaptor_runtime/_background/backend_runner.py +++ b/src/openjd/adaptor_runtime/_background/backend_runner.py @@ -116,7 +116,9 @@ def run(self, *, on_connection_file_written: List[Callable[[], None]] | None = N raise try: - with secure_open(self._connection_file_path, open_mode="w") as conn_file: + with secure_open( + self._connection_file_path, open_mode="w", encoding="utf-8" + ) as conn_file: json.dump( ConnectionSettings(server_path), conn_file, diff --git a/src/openjd/adaptor_runtime/_background/frontend_runner.py b/src/openjd/adaptor_runtime/_background/frontend_runner.py index c83b4ba5..e47d4dc3 100644 --- a/src/openjd/adaptor_runtime/_background/frontend_runner.py +++ b/src/openjd/adaptor_runtime/_background/frontend_runner.py @@ -153,7 +153,7 @@ def init( bootstrap_output_path = os.path.join( bootstrap_log_dir, f"adaptor-runtime-background-bootstrap-output-{bootstrap_id}.log" ) - output_log_file = open(bootstrap_output_path, mode="w+") + output_log_file = open(bootstrap_output_path, mode="w+", encoding="utf-8") try: process = subprocess.Popen( args, @@ -194,7 +194,7 @@ def init( if process.stderr: process.stderr.close() - with open(bootstrap_output_path, mode="r") as f: + with open(bootstrap_output_path, mode="r", encoding="utf-8") as f: bootstrap_output = f.readlines() _logger.info("========== BEGIN BOOTSTRAP OUTPUT CONTENTS ==========") for line in bootstrap_output: @@ -203,7 +203,7 @@ def init( _logger.info(f"Checking for bootstrap logs at '{bootstrap_log_path}'") try: - with open(bootstrap_log_path, mode="r") as f: + with open(bootstrap_log_path, mode="r", encoding="utf-8") as f: bootstrap_logs = f.readlines() except Exception as e: _logger.error(f"Failed to get bootstrap logs at '{bootstrap_log_path}': {e}") @@ -442,7 +442,8 @@ def _wait_for_connection_file( def file_is_openable() -> bool: try: - open(filepath, mode="r").close() + with open(filepath, mode="r", encoding="utf-8"): + pass except IOError: # File is not available yet return False diff --git a/src/openjd/adaptor_runtime/_background/loaders.py b/src/openjd/adaptor_runtime/_background/loaders.py index e258a291..1ccd7264 100644 --- a/src/openjd/adaptor_runtime/_background/loaders.py +++ b/src/openjd/adaptor_runtime/_background/loaders.py @@ -35,7 +35,7 @@ class ConnectionSettingsFileLoader(ConnectionSettingsLoader): def load(self) -> ConnectionSettings: try: - with open(self.file_path) as conn_file: + with open(self.file_path, encoding="utf-8") as conn_file: loaded_settings = json.load(conn_file) except OSError as e: errmsg = f"Failed to open connection file '{self.file_path}': {e}" diff --git a/src/openjd/adaptor_runtime/_background/log_buffers.py b/src/openjd/adaptor_runtime/_background/log_buffers.py index 1aa3db61..394c1bd5 100644 --- a/src/openjd/adaptor_runtime/_background/log_buffers.py +++ b/src/openjd/adaptor_runtime/_background/log_buffers.py @@ -132,7 +132,7 @@ def __init__(self, filepath: str, *, formatter: logging.Formatter | None = None) def buffer(self, record: logging.LogRecord) -> None: with ( self._file_lock, - secure_open(self._filepath, open_mode="a") as f, + secure_open(self._filepath, open_mode="a", encoding="utf-8") as f, ): f.write(self._format(record)) @@ -142,7 +142,7 @@ def chunk(self) -> BufferedOutput: with ( self._chunk_lock, self._file_lock, - open(self._filepath, mode="r") as f, + open(self._filepath, mode="r", encoding="utf-8") as f, ): self._chunk.id = id f.seek(self._chunk.start) diff --git a/src/openjd/adaptor_runtime/_entrypoint.py b/src/openjd/adaptor_runtime/_entrypoint.py index 23ad98dd..df1a906f 100644 --- a/src/openjd/adaptor_runtime/_entrypoint.py +++ b/src/openjd/adaptor_runtime/_entrypoint.py @@ -179,6 +179,10 @@ def _init_loggers(self, *, bootstrap_log_path: str | None = None) -> _LogConfig: formatter = ConditionalFormatter( "%(levelname)s: %(message)s", ignore_patterns=[_OPENJD_LOG_REGEX] ) + + # Ensure stdout uses UTF-8 encoding to avoid issues with + # encoding Non-ASCII characters. + sys.stdout.reconfigure(encoding="utf-8") # type: ignore stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setFormatter(formatter) @@ -590,14 +594,14 @@ def _load_data(data: str) -> dict: def _load_yaml_json(data: str) -> Any: """ - Loads a YAML/JSON file/string. + Loads a YAML/JSON file/string using the UTF-8 encoding. Note that yaml.safe_load() is capable of loading JSON documents. """ loaded_yaml = None if data.startswith("file://"): filepath = data[len("file://") :] - with open(filepath) as yaml_file: + with open(filepath, encoding="utf-8") as yaml_file: loaded_yaml = yaml.safe_load(yaml_file) else: loaded_yaml = yaml.safe_load(data) diff --git a/src/openjd/adaptor_runtime/adaptors/_validator.py b/src/openjd/adaptor_runtime/adaptors/_validator.py index cccd746f..7f292e49 100644 --- a/src/openjd/adaptor_runtime/adaptors/_validator.py +++ b/src/openjd/adaptor_runtime/adaptors/_validator.py @@ -74,7 +74,7 @@ def from_schema_file(schema_path: str) -> AdaptorDataValidator: schema_path (str): The path to the JSON schema file to use. """ try: - with open(schema_path) as schema_file: + with open(schema_path, encoding="utf-8") as schema_file: schema = json.load(schema_file) except json.JSONDecodeError as e: _logger.error(f"Failed to decode JSON schema file: {e}") @@ -148,7 +148,7 @@ def _load_yaml_json(data: str) -> Any: loaded_yaml = None if data.startswith("file://"): filepath = data[len("file://") :] - with open(filepath) as yaml_file: + with open(filepath, encoding="utf-8") as yaml_file: loaded_yaml = yaml.safe_load(yaml_file) else: loaded_yaml = yaml.safe_load(data) diff --git a/src/openjd/adaptor_runtime/adaptors/configuration/_configuration.py b/src/openjd/adaptor_runtime/adaptors/configuration/_configuration.py index 320ffbb9..92b015ff 100644 --- a/src/openjd/adaptor_runtime/adaptors/configuration/_configuration.py +++ b/src/openjd/adaptor_runtime/adaptors/configuration/_configuration.py @@ -82,7 +82,8 @@ def from_file( """ try: - config = json.load(open(config_path)) + with open(config_path, encoding="utf-8") as config_file: + config = json.load(config_file) except OSError as e: _logger.error(f"Failed to open configuration at {config_path}: {e}") raise @@ -102,7 +103,8 @@ def from_file( schema_paths = schema_path if isinstance(schema_path, list) else [schema_path] for path in schema_paths: try: - schema = json.load(open(path)) + with open(path, encoding="utf-8") as schema_file: + schema = json.load(schema_file) except OSError as e: _logger.error(f"Failed to open configuration schema at {path}: {e}") raise diff --git a/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py b/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py index b5f8e0ef..95b0205a 100644 --- a/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py +++ b/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py @@ -95,7 +95,7 @@ def _ensure_config_file(filepath: str, *, create: bool = False) -> bool: _logger.info(f"Creating empty configuration at {filepath}") try: os.makedirs(os.path.dirname(filepath), mode=stat.S_IRWXU, exist_ok=True) - with secure_open(filepath, open_mode="w") as f: + with secure_open(filepath, open_mode="w", encoding="utf-8") as f: json.dump({}, f) except OSError as e: _logger.warning(f"Could not write empty configuration to {filepath}: {e}") diff --git a/test/openjd/adaptor_runtime/integ/_utils/test_secure_open.py b/test/openjd/adaptor_runtime/integ/_utils/test_secure_open.py index 2508ad49..8f28840a 100644 --- a/test/openjd/adaptor_runtime/integ/_utils/test_secure_open.py +++ b/test/openjd/adaptor_runtime/integ/_utils/test_secure_open.py @@ -24,7 +24,7 @@ def create_file(): f"secure_open_test_{''.join(random.choice(string.ascii_letters) for _ in range(10))}.txt" ) test_file_path = os.path.join(tempfile.gettempdir(), test_file_name) - with secure_open(test_file_path, open_mode="w") as test_file: + with secure_open(test_file_path, open_mode="w", encoding="utf-8") as test_file: test_file.write(file_content) yield test_file_path, file_content os.remove(test_file_path) @@ -36,7 +36,7 @@ def test_secure_open_write_and_read(self, create_file): Test if the file owner can write and read the file """ test_file_path, file_content = create_file - with secure_open(test_file_path, open_mode="r") as test_file: + with secure_open(test_file_path, open_mode="r", encoding="utf-8") as test_file: result = test_file.read() assert result == file_content @@ -61,7 +61,7 @@ def test_secure_open_file_windows_permission(self, create_file, win_test_user): try: with pytest.raises(PermissionError): - with open(test_file_path, "r") as f: + with open(test_file_path, "r", encoding="utf-8") as f: f.read() finally: # Revert the impersonation diff --git a/test/openjd/adaptor_runtime/integ/background/test_background_mode.py b/test/openjd/adaptor_runtime/integ/background/test_background_mode.py index 974e7361..5357f482 100644 --- a/test/openjd/adaptor_runtime/integ/background/test_background_mode.py +++ b/test/openjd/adaptor_runtime/integ/background/test_background_mode.py @@ -46,7 +46,7 @@ def mock_runtime_logger_level(self, tmpdir: pathlib.Path): # Set up a config file for the backend process config = {"log_level": "DEBUG"} config_path = os.path.join(tmpdir, "configuration.json") - with open(config_path, mode="w") as f: + with open(config_path, mode="w", encoding="utf-8") as f: json.dump(config, f) # Override the default config path to the one we just created diff --git a/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration.py b/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration.py index 3deeabde..80660019 100644 --- a/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration.py +++ b/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration.py @@ -5,7 +5,7 @@ from json.decoder import JSONDecodeError from typing import Any from typing import List as _List -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, call, patch, ANY import jsonschema import pytest @@ -55,7 +55,18 @@ def test_loads_schema( result = Configuration.from_file(config_path, schema_path) # THEN - mock_open.assert_has_calls([call(config_path), call(schema_path)]) + mock_open.assert_has_calls( + [ + call(config_path, encoding="utf-8"), + # The __enter__ and __exit__ are context manager + # calls for opening files safely. + call().__enter__(), + call().__exit__(None, None, None), + call(schema_path, encoding="utf-8"), + call().__enter__(), + call().__exit__(None, None, None), + ] + ) assert mock_load.call_count == 2 mock_validate.assert_called_once_with(config, schema) assert result._config is config @@ -77,7 +88,7 @@ def test_skips_validation_when_no_schema( result = Configuration.from_file(config_path) # THEN - mock_open.assert_called_once_with(config_path) + mock_open.assert_called_once_with(config_path, encoding="utf-8") mock_load.assert_called_once() assert ( f"JSON Schema file path not provided. Configuration file {config_path} will not be " @@ -107,7 +118,21 @@ def test_validates_against_multiple_schemas( result = Configuration.from_file(config_path, [schema_path, schema2_path]) # THEN - mock_open.assert_has_calls([call(config_path), call(schema_path), call(schema2_path)]) + mock_open.assert_has_calls( + [ + call(config_path, encoding="utf-8"), + # The __enter__ and __exit__ are context manager + # calls for opening files safely. + call().__enter__(), + call().__exit__(None, None, None), + call(schema_path, encoding="utf-8"), + call().__enter__(), + call().__exit__(None, None, None), + call(schema2_path, encoding="utf-8"), + call().__enter__(), + call().__exit__(None, None, None), + ] + ) assert mock_load.call_count == 3 mock_validate.assert_has_calls([call(config, schema), call(config, schema2)]) assert result._config is config @@ -128,7 +153,7 @@ def test_raises_when_nonvalid_schema_path_value( # THEN mock_load.assert_called_once() - mock_open.assert_called_once_with(config_path) + mock_open.assert_called_once_with(config_path, encoding="utf-8") assert raised_err.match(f"Schema path cannot be an empty {type(schema_path)}") @patch.object(configuration.json, "load") @@ -148,7 +173,9 @@ def test_raises_when_schema_open_fails( # THEN mock_load.assert_called_once() - mock_open.assert_has_calls([call(config_path), call(schema_path)]) + mock_open.assert_has_calls( + [call(config_path, encoding="utf-8"), call(schema_path, encoding="utf-8")] + ) assert raised_err.value is err assert f"Failed to open configuration schema at {schema_path}: " in caplog.text @@ -169,7 +196,18 @@ def test_raises_when_schema_json_decode_fails( # THEN assert mock_load.call_count == 2 - mock_open.assert_has_calls([call(config_path), call(schema_path)]) + mock_open.assert_has_calls( + [ + call(config_path, encoding="utf-8"), + # The __enter__ and __exit__ are context manager + # calls for opening files safely. + call().__enter__(), + call().__exit__(None, None, None), + call(schema_path, encoding="utf-8"), + call().__enter__(), + call().__exit__(JSONDecodeError, ANY, ANY), + ] + ) assert raised_err.value is err assert f"Failed to decode configuration schema at {schema_path}: " in caplog.text @@ -187,7 +225,7 @@ def test_raises_when_config_open_fails( Configuration.from_file(config_path, "") # THEN - mock_open.assert_called_once_with(config_path) + mock_open.assert_called_once_with(config_path, encoding="utf-8") assert raised_err.value is err assert f"Failed to open configuration at {config_path}: " in caplog.text @@ -206,7 +244,7 @@ def test_raises_when_config_json_decode_fails( Configuration.from_file(config_path, "") # THEN - mock_open.assert_called_once_with(config_path) + mock_open.assert_called_once_with(config_path, encoding="utf-8") mock_load.assert_called_once() assert raised_err.value is err assert f"Failed to decode configuration at {config_path}: " in caplog.text @@ -234,7 +272,18 @@ def test_raises_when_config_fails_jsonschema_validation( Configuration.from_file(config_path, schema_path) # THEN - mock_open.assert_has_calls([call(config_path), call(schema_path)]) + mock_open.assert_has_calls( + [ + call(config_path, encoding="utf-8"), + # The __enter__ and __exit__ are context manager + # calls for opening files safely. + call().__enter__(), + call().__exit__(None, None, None), + call(schema_path, encoding="utf-8"), + call().__enter__(), + call().__exit__(None, None, None), + ] + ) assert mock_load.call_count == 2 mock_validate.assert_called_once_with(config, schema) assert raised_err.value is mock_validate.side_effect diff --git a/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration_manager.py b/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration_manager.py index f48c436f..8146c40c 100644 --- a/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration_manager.py +++ b/test/openjd/adaptor_runtime/unit/adaptors/configuration/test_configuration_manager.py @@ -119,7 +119,7 @@ def test_create( # THEN assert result == created mock_exists.assert_called_once_with(path) - open_mock.assert_called_once_with(path, open_mode="w") + open_mock.assert_called_once_with(path, open_mode="w", encoding="utf-8") assert f'Configuration file at "{path}" does not exist.' in caplog.text assert f"Creating empty configuration at {path}" in caplog.text if created: diff --git a/test/openjd/adaptor_runtime/unit/background/test_backend_runner.py b/test/openjd/adaptor_runtime/unit/background/test_backend_runner.py index 1ce62a6a..9b52eede 100644 --- a/test/openjd/adaptor_runtime/unit/background/test_backend_runner.py +++ b/test/openjd/adaptor_runtime/unit/background/test_backend_runner.py @@ -96,7 +96,7 @@ def test_run( ) mock_thread.assert_called_once() mock_thread.return_value.start.assert_called_once() - open_mock.assert_called_once_with(conn_file, open_mode="w") + open_mock.assert_called_once_with(conn_file, open_mode="w", encoding="utf-8") mock_json_dump.assert_called_once_with( ConnectionSettings(socket_path), open_mock.return_value, @@ -170,7 +170,7 @@ def test_run_raises_when_writing_connection_file_fails( ] mock_thread.assert_called_once() mock_thread.return_value.start.assert_called_once() - open_mock.assert_called_once_with(conn_file, open_mode="w") + open_mock.assert_called_once_with(conn_file, open_mode="w", encoding="utf-8") mock_thread.return_value.join.assert_called_once() if OSName.is_posix(): mock_os_remove.assert_has_calls([call(conn_file), call(socket_path)]) diff --git a/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py b/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py index 9545ea5c..0bb19e83 100644 --- a/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py +++ b/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py @@ -963,7 +963,7 @@ def test_waits_for_file( # THEN mock_exists.assert_has_calls([call(filepath)] * 2) mock_sleep.assert_has_calls([call(interval)] * 3) - open_mock.assert_has_calls([call(filepath, mode="r")] * 2) + open_mock.assert_has_calls([call(filepath, mode="r", encoding="utf-8")] * 2) @patch.object(frontend_runner.time, "sleep") @patch.object(frontend_runner.os.path, "exists") diff --git a/test/openjd/adaptor_runtime/unit/background/test_log_buffers.py b/test/openjd/adaptor_runtime/unit/background/test_log_buffers.py index 7ccb7c30..56da529b 100644 --- a/test/openjd/adaptor_runtime/unit/background/test_log_buffers.py +++ b/test/openjd/adaptor_runtime/unit/background/test_log_buffers.py @@ -120,7 +120,7 @@ def test_buffer(self) -> None: buffer.buffer(mock_record) # THEN - open_mock.assert_called_once_with(filepath, open_mode="a") + open_mock.assert_called_once_with(filepath, open_mode="a", encoding="utf-8") handle = open_mock.return_value handle.write.assert_called_once_with(mock_record.msg) @@ -140,7 +140,7 @@ def test_chunk(self, mocked_chunk_id: Tuple[str, MagicMock]) -> None: # THEN mock_create_id.assert_called_once() - open_mock.assert_called_once_with(filepath, mode="r") + open_mock.assert_called_once_with(filepath, mode="r", encoding="utf-8") handle = open_mock.return_value handle.seek.assert_called_once_with(buffer._chunk.start) handle.read.assert_called_once() diff --git a/test/openjd/adaptor_runtime/unit/test_entrypoint.py b/test/openjd/adaptor_runtime/unit/test_entrypoint.py index 46adc5d0..e859ac1d 100644 --- a/test/openjd/adaptor_runtime/unit/test_entrypoint.py +++ b/test/openjd/adaptor_runtime/unit/test_entrypoint.py @@ -865,7 +865,7 @@ def test_accepts_file(self, input: str, expected: dict): # THEN assert output == expected - open_mock.assert_called_once_with(filepath) + open_mock.assert_called_once_with(filepath, encoding="utf-8") @patch.object(runtime_entrypoint, "open") def test_raises_on_os_error(self, mock_open: MagicMock, caplog: pytest.LogCaptureFixture): @@ -880,7 +880,7 @@ def test_raises_on_os_error(self, mock_open: MagicMock, caplog: pytest.LogCaptur # THEN assert raised_err.value is mock_open.side_effect - mock_open.assert_called_once_with(filepath) + mock_open.assert_called_once_with(filepath, encoding="utf-8") assert "Failed to open data file: " in caplog.text def test_raises_when_parsing_fails(self, caplog: pytest.LogCaptureFixture): diff --git a/test/openjd/test_copyright_header.py b/test/openjd/test_copyright_header.py index 23b22dbb..8689a77d 100644 --- a/test/openjd/test_copyright_header.py +++ b/test/openjd/test_copyright_header.py @@ -33,7 +33,7 @@ def _check_file(filename: Path) -> None: def _is_version_file(filename: Path) -> bool: if filename.name != "_version.py": return False - with open(filename) as infile: + with open(filename, encoding="utf-8") as infile: lines_read = 0 for line in infile: if _generated_by_scm.search(line): From e93ef36c719924a0e01fed3f1c061c9c0a2de207 Mon Sep 17 00:00:00 2001 From: client-software-ci <129794699+client-software-ci@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:20:04 -0600 Subject: [PATCH 11/39] chore(release): 0.9.0 (#173) Signed-off-by: client-software-ci <129794699+client-software-ci@users.noreply.github.com> Signed-off-by: Justin Blagden --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de9526a7..69575612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 0.9.0 (2025-01-02) + + + +### Bug Fixes +* Ensure all open uses UTF-8 instead of default encoding. (#171) ([`74fe730`](https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/commit/74fe7304d8f1033414cda1b0d4a14fec3ee80fa0)) + ## 0.8.2 (2024-11-29) This release allows pywin32 version 307 and above to be used as a dependency for this package. Previously, only pywin32 version 308 was allowed. From a87840a432db8548ff03229fb768c43e2825493b Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Tue, 21 Jan 2025 14:53:03 -0600 Subject: [PATCH 12/39] feat! Adds user-friendly messaging when the adaptor executable isn't found Also checks the result of `which`, just in case there's an alias that's pointing to nowhere. Signed-off-by: Justin Blagden --- .../process/_logging_subprocess.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index 8376446c..ef874d7b 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging +import shutil import signal import subprocess import uuid @@ -21,7 +22,11 @@ class LoggingSubprocess(object): - """A process whose stdout/stderr lines are sent to a configurable logger""" + """A process whose stdout/stderr lines are sent to a configurable logger + + Raises: + FileNotFoundError: When the executable the adaptor is set to run cannot be found. + """ _logger: logging.Logger _process: subprocess.Popen @@ -64,7 +69,30 @@ def __init__( # In Windows, this is required for signal. SIGBREAK will be sent to the entire process group. # Without this one, current process will also get the SIGBREAK and may react incorrectly. popen_params.update(creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) # type: ignore[attr-defined] - self._process = subprocess.Popen(**popen_params) + + try: + self._process = subprocess.Popen(**popen_params) + + except FileNotFoundError as fnf_error: + # In ManagedProcess we prepend the executable to the list of arguments before creating a LoggingSubprocess + executable = args[0] + exe_path = shutil.which(executable) + + # If we didn't find the executable found by which + if type(exe_path) is not None: + raise FileNotFoundError( + f"Could not find adaptor executable at: {exe_path} using alias {executable}\n" + f"Error:{fnf_error}" + ) + + raise FileNotFoundError( + f"Could not find the executable associated with the adaptor: {executable}\n" + f"Is the executable on the PATH or in the startup directory?\n" + f"Error:{fnf_error}" + ) + + except Exception as error: + raise error if not self._process.stdout: # pragma: no cover raise RuntimeError("process stdout not set") From 4c89a0ad19d1a33a43a14a277e53c01322971d12 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Tue, 28 Jan 2025 09:22:51 -0600 Subject: [PATCH 13/39] Resolves code smells called out by sonarqube Fixed an always-true type check Removed an explicit throw of un-caught exception types Signed-off-by: Justin Blagden --- src/openjd/adaptor_runtime/process/_logging_subprocess.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index ef874d7b..fdbee8f7 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -79,7 +79,7 @@ def __init__( exe_path = shutil.which(executable) # If we didn't find the executable found by which - if type(exe_path) is not None: + if exe_path is not None: raise FileNotFoundError( f"Could not find adaptor executable at: {exe_path} using alias {executable}\n" f"Error:{fnf_error}" @@ -91,9 +91,6 @@ def __init__( f"Error:{fnf_error}" ) - except Exception as error: - raise error - if not self._process.stdout: # pragma: no cover raise RuntimeError("process stdout not set") if not self._process.stderr: # pragma: no cover From d800bbd5726f067d0c9acb87743821a1e052fa40 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Mon, 3 Feb 2025 13:25:14 -0600 Subject: [PATCH 14/39] Adds a test for executable not found error Signed-off-by: Justin Blagden --- .../process/test_integration_logging_subprocess.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py b/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py index c524104a..474ccabb 100644 --- a/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py +++ b/test/openjd/adaptor_runtime/integ/process/test_integration_logging_subprocess.py @@ -113,9 +113,9 @@ def test_startup_directory(self, startup_dir: str | None, caplog): def test_startup_directory_empty_posix(self): """When calling LoggingSubprocess with an empty cwd, FileNotFoundError will be raised.""" args = ["pwd"] - with pytest.raises(FileNotFoundError) as excinfo: + with pytest.raises(FileNotFoundError) as exc_info: LoggingSubprocess(args=args, startup_directory="") - assert "[Errno 2] No such file or directory: ''" in str(excinfo.value) + assert "[Errno 2] No such file or directory: ''" in str(exc_info.value) @pytest.mark.skipif(not OSName.is_windows(), reason="Only run this test in Windows.") def test_startup_directory_empty_windows(self): @@ -151,6 +151,13 @@ def test_log_levels(self, log_level: int, caplog): assert any(r.message == message and r.levelno == _STDERR_LEVEL for r in records) + def test_executable_not_found(self): + """When calling LoggingSubprocess with a missing executable, FileNotFoundError will be raised""" + args = ["missing_executable"] + with pytest.raises(FileNotFoundError) as exc_info: + LoggingSubprocess(args=args) + assert "Could not find the executable associated with the adaptor" in str(exc_info.value) + class TestIntegrationRegexHandler(object): """Integration tests for LoggingSubprocess""" From e34036d0ab52dc19892f50bde7ec02bc57892dd8 Mon Sep 17 00:00:00 2001 From: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:48:01 -0600 Subject: [PATCH 15/39] chore: update GitHub issue templates (#176) Signed-off-by: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- .github/ISSUE_TEMPLATE/bug.yml | 8 ++++---- .github/ISSUE_TEMPLATE/doc.yml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- .github/ISSUE_TEMPLATE/maintenance.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index f3e79e2c..9262a3b3 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -9,7 +9,7 @@ body: Thank you for taking the time to fill out this bug report! ⚠️ If the bug that you are reporting is a security-related issue or security vulnerability, - then please do not create a report via this template. Instead please + then please do not create a report via this template. Instead please notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to [AWS Security](aws-security@amazon.com). @@ -54,9 +54,9 @@ body: description: Please provide information on the environment and software versions that you are using to reproduce the bug. value: | At minimum: - 1. Operating system: (e.g. Windows Server 2022; Amazon Linux 2023; etc.) - 2. Output of `python3 --version`: - 3. Version of this library. + 1. Operating system (e.g. Windows Server 2022; Amazon Linux 2023; etc.) + 2. Output of `python3 --version` + 3. Version of this library Please share other details about your environment that you think might be relevant to reproducing the bug. validations: diff --git a/.github/ISSUE_TEMPLATE/doc.yml b/.github/ISSUE_TEMPLATE/doc.yml index 7a95fad6..cdde690d 100644 --- a/.github/ISSUE_TEMPLATE/doc.yml +++ b/.github/ISSUE_TEMPLATE/doc.yml @@ -2,7 +2,7 @@ name: "📕 Documentation Issue" description: Issue in the documentation title: "Docs: (short description of the issue)" -labels: ["documenation", "needs triage"] +labels: ["documentation", "needs triage"] body: - type: textarea id: documentation_issue diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index d2844682..d644b5f6 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,7 +1,7 @@ name: "\U0001F680 Feature Request" description: Request a new feature title: "Feature request: (short description of the feature)" -labels: ["feature", "needs triage"] +labels: ["enhancement", "needs triage"] body: - type: textarea id: problem diff --git a/.github/ISSUE_TEMPLATE/maintenance.yml b/.github/ISSUE_TEMPLATE/maintenance.yml index f2440eb5..a109a9a0 100644 --- a/.github/ISSUE_TEMPLATE/maintenance.yml +++ b/.github/ISSUE_TEMPLATE/maintenance.yml @@ -1,7 +1,7 @@ name: "🛠️ Maintenance" description: Some type of improvement title: "Maintenance: (short description of the issue)" -labels: ["feature", "needs triage"] +labels: ["needs triage"] body: - type: textarea id: description From 19849745de5350362734b6f5a6f99fb50fd3d49f Mon Sep 17 00:00:00 2001 From: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Date: Fri, 31 Jan 2025 09:56:49 -0600 Subject: [PATCH 16/39] chore: increase log level for daemon _serve args to info (#169) Signed-off-by: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- .../_background/frontend_runner.py | 2 +- .../integ/test_integration_entrypoint.py | 4 +-- .../unit/background/test_frontend_runner.py | 28 +++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/openjd/adaptor_runtime/_background/frontend_runner.py b/src/openjd/adaptor_runtime/_background/frontend_runner.py index e47d4dc3..d9bd8644 100644 --- a/src/openjd/adaptor_runtime/_background/frontend_runner.py +++ b/src/openjd/adaptor_runtime/_background/frontend_runner.py @@ -149,7 +149,7 @@ def init( ) args.extend(["--bootstrap-log-file", bootstrap_log_path]) - _logger.debug(f"Running process with args: {args}") + _logger.info(f"Running process with args: {args}") bootstrap_output_path = os.path.join( bootstrap_log_dir, f"adaptor-runtime-background-bootstrap-output-{bootstrap_id}.log" ) diff --git a/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py b/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py index d1690ff6..0218c7e4 100644 --- a/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py +++ b/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py @@ -112,8 +112,8 @@ def test_start_stop(self, caplog: pytest.LogCaptureFixture, tmp_path: Path): assert "Connected successfully" in caplog.text assert "Running in background daemon mode." in caplog.text assert "Daemon background process stopped." in caplog.text - assert "on_prerun" not in caplog.text - assert "on_postrun" not in caplog.text + assert "on_prerun" in caplog.text + assert "on_postrun" in caplog.text def test_run(self, caplog: pytest.LogCaptureFixture, tmp_path: Path): # GIVEN diff --git a/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py b/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py index 0bb19e83..64736f0a 100644 --- a/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py +++ b/test/openjd/adaptor_runtime/unit/background/test_frontend_runner.py @@ -219,6 +219,34 @@ def test_initializes_backend_process( ) mock_heartbeat.assert_called_once() + def test_arguments_to_daemon_serve_are_logged_at_info_level( + self, + mock_path_exists: MagicMock, + mock_Popen: MagicMock, + caplog: pytest.LogCaptureFixture, + ): + # GIVEN + caplog.set_level("INFO") + mock_path_exists.return_value = False + adaptor_module = ModuleType("") + adaptor_module.__package__ = "package" + conn_file_path = Path("/path") + runner = FrontendRunner() + + # WHEN + runner.init( + adaptor_module=adaptor_module, + connection_file_path=conn_file_path, + ) + + # THEN + assert any( + "Running process with args" in captured_message + for captured_message in caplog.messages + ) + mock_path_exists.assert_called_once_with() + mock_Popen.assert_called_once() + def test_raises_when_adaptor_module_not_package(self): # GIVEN adaptor_module = ModuleType("") From 5dd3862543d246548b7e0ddd36735f854c386b19 Mon Sep 17 00:00:00 2001 From: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:22:07 -0600 Subject: [PATCH 17/39] chore: add maintenance label to maintenance GitHub template (#182) Signed-off-by: Joel Wong <127782171+joel-wong-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- .github/ISSUE_TEMPLATE/maintenance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/maintenance.yml b/.github/ISSUE_TEMPLATE/maintenance.yml index a109a9a0..e95acbfc 100644 --- a/.github/ISSUE_TEMPLATE/maintenance.yml +++ b/.github/ISSUE_TEMPLATE/maintenance.yml @@ -1,7 +1,7 @@ name: "🛠️ Maintenance" description: Some type of improvement title: "Maintenance: (short description of the issue)" -labels: ["needs triage"] +labels: ["maintenance", "needs triage"] body: - type: textarea id: description From 77104bc880fb5af7934184a2e3bf2d8096a80b62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Feb 2025 11:15:14 -0600 Subject: [PATCH 18/39] chore(deps): update python-semantic-release requirement (#181) Updates the requirements on [python-semantic-release](https://github.com/python-semantic-release/python-semantic-release) to permit the latest version. - [Release notes](https://github.com/python-semantic-release/python-semantic-release/releases) - [Changelog](https://github.com/python-semantic-release/python-semantic-release/blob/master/CHANGELOG.rst) - [Commits](https://github.com/python-semantic-release/python-semantic-release/compare/v9.14...v9.20) --- updated-dependencies: - dependency-name: python-semantic-release dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-release.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-release.txt b/requirements-release.txt index c49048b3..3c2e488b 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1 +1 @@ -python-semantic-release == 9.14.* \ No newline at end of file +python-semantic-release == 9.20.* \ No newline at end of file From 1b0de05897d107807dce60697d519d1b85cece84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Feb 2025 11:21:29 -0600 Subject: [PATCH 19/39] chore(deps): update hatch requirement from ==1.13.* to ==1.14.* (#168) Updates the requirements on [hatch](https://github.com/pypa/hatch) to permit the latest version. - [Release notes](https://github.com/pypa/hatch/releases) - [Commits](https://github.com/pypa/hatch/compare/hatch-v1.13.0...hatch-v1.14.0) --- updated-dependencies: - dependency-name: hatch dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-development.txt b/requirements-development.txt index 2d27088d..5997871b 100644 --- a/requirements-development.txt +++ b/requirements-development.txt @@ -1,2 +1,2 @@ -hatch == 1.13.* +hatch == 1.14.* hatch-vcs == 0.4.* \ No newline at end of file From f4d77938581781f6329f4dfa04ca0f78ef6f95df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Feb 2025 11:47:09 -0600 Subject: [PATCH 20/39] chore(deps): update ruff requirement from ==0.8.* to ==0.9.* (#175) Updates the requirements on [ruff](https://github.com/astral-sh/ruff) to permit the latest version. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.8.0...0.9.1) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 69e723e6..be3da1d0 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -5,7 +5,7 @@ pytest-cov == 6.0.* pytest-timeout == 2.3.* pytest-xdist == 3.6.* black == 24.* -ruff == 0.8.* +ruff == 0.9.* mypy == 1.11.* psutil == 6.1.* types-PyYAML ~= 6.0 From ea7852a065a63a1cbbe9e98b633a1b307345e4e8 Mon Sep 17 00:00:00 2001 From: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Date: Wed, 26 Feb 2025 09:56:48 -0600 Subject: [PATCH 21/39] test: Broaden regex to ignore copyright header file created by setuptools (#186) Signed-off-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- test/openjd/test_copyright_header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/openjd/test_copyright_header.py b/test/openjd/test_copyright_header.py index 8689a77d..129ab741 100644 --- a/test/openjd/test_copyright_header.py +++ b/test/openjd/test_copyright_header.py @@ -7,7 +7,7 @@ _copyright_header_re = re.compile( r"Copyright Amazon\.com, Inc\. or its affiliates\. All Rights Reserved\.", re.IGNORECASE ) -_generated_by_scm = re.compile(r"# file generated by setuptools_scm", re.IGNORECASE) +_generated_by_scm = re.compile(r"# file generated by setuptools[_-]scm", re.IGNORECASE) def _check_file(filename: Path) -> None: From d554197198d3c14116fdb7a7d220c673a2d8a614 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Feb 2025 10:11:17 -0600 Subject: [PATCH 22/39] chore(deps): update psutil requirement from ==6.1.* to ==7.0.* (#185) Updates the requirements on [psutil](https://github.com/giampaolo/psutil) to permit the latest version. - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-6.1.0...release-7.0.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index be3da1d0..6a8bd7e7 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -7,5 +7,5 @@ pytest-xdist == 3.6.* black == 24.* ruff == 0.9.* mypy == 1.11.* -psutil == 6.1.* +psutil == 7.0.* types-PyYAML ~= 6.0 From 10e9aa1c55105494f5380c191f11af89106089a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Feb 2025 10:23:36 -0600 Subject: [PATCH 23/39] chore(deps): update black requirement from ==24.* to ==25.* (#184) Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.1a1...25.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 6a8bd7e7..ad57808b 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -4,7 +4,7 @@ pytest == 7.4.* pytest-cov == 6.0.* pytest-timeout == 2.3.* pytest-xdist == 3.6.* -black == 24.* +black == 25.* ruff == 0.9.* mypy == 1.11.* psutil == 7.0.* From 5ad65016f04d90e6cf1011969a10f5fcf9931d53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Feb 2025 10:28:52 -0600 Subject: [PATCH 24/39] chore(deps): update python-semantic-release requirement (#183) Updates the requirements on [python-semantic-release](https://github.com/python-semantic-release/python-semantic-release) to permit the latest version. - [Release notes](https://github.com/python-semantic-release/python-semantic-release/releases) - [Changelog](https://github.com/python-semantic-release/python-semantic-release/blob/master/CHANGELOG.rst) - [Commits](https://github.com/python-semantic-release/python-semantic-release/compare/v9.20...v9.21) --- updated-dependencies: - dependency-name: python-semantic-release dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-release.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-release.txt b/requirements-release.txt index 3c2e488b..417ade73 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1 +1 @@ -python-semantic-release == 9.20.* \ No newline at end of file +python-semantic-release == 9.21.* \ No newline at end of file From 6fe101821877901abb7899805807be0cdc743d2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Mar 2025 09:29:02 -0500 Subject: [PATCH 25/39] chore(deps): update ruff requirement from ==0.9.* to ==0.11.* (#189) Updates the requirements on [ruff](https://github.com/astral-sh/ruff) to permit the latest version. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.9.0...0.11.0) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index ad57808b..78dabae6 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -5,7 +5,7 @@ pytest-cov == 6.0.* pytest-timeout == 2.3.* pytest-xdist == 3.6.* black == 25.* -ruff == 0.9.* +ruff == 0.11.* mypy == 1.11.* psutil == 7.0.* types-PyYAML ~= 6.0 From 313d2abf1f602ae68017d0ae15b4c076d5012b7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Apr 2025 09:20:07 -0700 Subject: [PATCH 26/39] chore(deps): update pytest-cov requirement from ==6.0.* to ==6.1.* (#190) Updates the requirements on [pytest-cov](https://github.com/pytest-dev/pytest-cov) to permit the latest version. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v6.0.0...v6.1.1) --- updated-dependencies: - dependency-name: pytest-cov dependency-version: 6.1.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 78dabae6..89a448b5 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,7 +1,7 @@ coverage[toml] == 7.* coverage-conditional-plugin == 0.9.0 pytest == 7.4.* -pytest-cov == 6.0.* +pytest-cov == 6.1.* pytest-timeout == 2.3.* pytest-xdist == 3.6.* black == 25.* From 574ba15fea84539590ceda317c6190c81a6627a9 Mon Sep 17 00:00:00 2001 From: Cody Edwards <133044276+edwards-aws@users.noreply.github.com> Date: Fri, 11 Apr 2025 10:08:44 -0500 Subject: [PATCH 27/39] chore: changes for mypy update (#191) Signed-off-by: Cody Edwards Signed-off-by: Justin Blagden --- hatch.toml | 2 +- requirements-testing.txt | 2 +- .../adaptor_runtime/_background/model.py | 17 +++++---- .../configuration/_configuration_manager.py | 3 +- .../process/_logging_subprocess.py | 1 + .../unit/background/test_model.py | 36 ++++++++++++++++--- .../test_integration_client_interface.py | 2 ++ 7 files changed, 46 insertions(+), 17 deletions(-) diff --git a/hatch.toml b/hatch.toml index 3d5d8dd3..2fcb32f6 100644 --- a/hatch.toml +++ b/hatch.toml @@ -5,7 +5,7 @@ pre-install-commands = [ [envs.default.scripts] sync = "pip install -r requirements-testing.txt" -test = "pytest --cov-config pyproject.toml {args:test}" +test = "pytest --cov-config pyproject.toml {args:test} -vv" typing = "mypy {args:src test}" style = [ "ruff check {args:.}", diff --git a/requirements-testing.txt b/requirements-testing.txt index 89a448b5..b1fda23e 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -6,6 +6,6 @@ pytest-timeout == 2.3.* pytest-xdist == 3.6.* black == 25.* ruff == 0.11.* -mypy == 1.11.* +mypy == 1.15.* psutil == 7.0.* types-PyYAML ~= 6.0 diff --git a/src/openjd/adaptor_runtime/_background/model.py b/src/openjd/adaptor_runtime/_background/model.py index 18b06d35..1f2ca30b 100644 --- a/src/openjd/adaptor_runtime/_background/model.py +++ b/src/openjd/adaptor_runtime/_background/model.py @@ -3,7 +3,8 @@ import dataclasses as dataclasses import json as json from enum import Enum as Enum -from typing import Any, ClassVar, Dict, Generic, Iterable, Type, TypeVar, cast +from enum import EnumMeta +from typing import Any, ClassVar, Dict, Type, TypeVar, cast, Generic from ..adaptors import AdaptorState @@ -100,14 +101,12 @@ def map(self, o: Dict) -> _T: value = o[field.name] if dataclasses.is_dataclass(field.type): - value = DataclassMapper(field.type).map(value) - elif issubclass(field.type, Enum): - [value] = [ - enum - # Need to cast here for mypy - for enum in cast(Iterable[Enum], list(field.type)) - if enum.value == value - ] + # The init function expects a type, so any dataclasses in cls + # will be a type. + value = DataclassMapper(cast(type, field.type)).map(value) + elif isinstance(field.type, EnumMeta): + value = field.type(value) + args[field.name] = value return self._cls(**args) diff --git a/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py b/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py index 95b0205a..2eb5e5d8 100644 --- a/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py +++ b/src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py @@ -23,10 +23,11 @@ _ConfigType = TypeVar("_ConfigType", bound=Configuration) _AdaptorConfigType = TypeVar("_AdaptorConfigType", bound=AdaptorConfiguration) +_AdaptorConfigClassType = Type[_AdaptorConfigType] def create_adaptor_configuration_manager( - config_cls: Type[_AdaptorConfigType], + config_cls: _AdaptorConfigClassType, adaptor_name: str, default_config_path: str, schema_path: str | List[str] | None = None, diff --git a/src/openjd/adaptor_runtime/process/_logging_subprocess.py b/src/openjd/adaptor_runtime/process/_logging_subprocess.py index fdbee8f7..cb74c290 100644 --- a/src/openjd/adaptor_runtime/process/_logging_subprocess.py +++ b/src/openjd/adaptor_runtime/process/_logging_subprocess.py @@ -202,6 +202,7 @@ def terminate(self, grace_time_s: float = 60) -> None: self._process.kill() self._process.wait() else: + signal_type: signal.Signals if OSName.is_windows(): # pragma: is-posix # We use `CREATE_NEW_PROCESS_GROUP` to create the process, # so pid here is also the process group id and SIGBREAK can be only sent to the process group. diff --git a/test/openjd/adaptor_runtime/unit/background/test_model.py b/test/openjd/adaptor_runtime/unit/background/test_model.py index 3a74b447..e47fc2b1 100644 --- a/test/openjd/adaptor_runtime/unit/background/test_model.py +++ b/test/openjd/adaptor_runtime/unit/background/test_model.py @@ -2,11 +2,22 @@ import dataclasses +from enum import Enum + import pytest from openjd.adaptor_runtime._background.model import DataclassMapper +class StrEnum(str, Enum): + TEST = "test_str" + TEST_UNI = "test_☃" + + +class NormalEnum(Enum): + ONE = 1 + + # Define two dataclasses to use for tests @dataclasses.dataclass class Inner: @@ -17,6 +28,9 @@ class Inner: class Outer: outer_key: str inner: Inner + test_str: StrEnum + test_str_unicode: StrEnum + normal_enum: NormalEnum class TestDataclassMapper: @@ -26,17 +40,29 @@ class TestDataclassMapper: def test_maps_nested_dataclass(self): # GIVEN - input = {"outer_key": "outer_value", "inner": {"key": "value"}} + input = { + "outer_key": "outer_value", + "inner": { + "key": "value", + }, + "test_str": "test_str", + "test_str_unicode": "test_☃", + "normal_enum": 1, + } mapper = DataclassMapper(Outer) # WHEN result = mapper.map(input) # THEN - assert isinstance(result, Outer) - assert isinstance(result.inner, Inner) - assert result.outer_key == "outer_value" - assert result.inner.key == "value" + expected_dataclass = Outer( + outer_key="outer_value", + inner=Inner(key="value"), + test_str=StrEnum.TEST, + test_str_unicode=StrEnum.TEST_UNI, + normal_enum=NormalEnum.ONE, + ) + assert result == expected_dataclass def test_raises_when_field_is_missing(self): # GIVEN diff --git a/test/openjd/adaptor_runtime_client/integ/test_integration_client_interface.py b/test/openjd/adaptor_runtime_client/integ/test_integration_client_interface.py index 1dd09405..98490a73 100644 --- a/test/openjd/adaptor_runtime_client/integ/test_integration_client_interface.py +++ b/test/openjd/adaptor_runtime_client/integ/test_integration_client_interface.py @@ -35,6 +35,7 @@ def test_graceful_shutdown(self) -> None: # To avoid a race condition, giving some extra time for the logging subprocess to start. _sleep(0.5 if OSName.is_posix() else 4) + signal_type: signal.Signals if OSName.is_windows(): signal_type = signal.CTRL_BREAK_EVENT # type: ignore[attr-defined] else: @@ -77,6 +78,7 @@ def test_client_in_thread_does_not_do_graceful_shutdown(self) -> None: # To avoid a race condition, giving some extra time for the logging subprocess to start. _sleep(0.5 if OSName.is_posix() else 4) + signal_type: signal.Signals if OSName.is_windows(): signal_type = signal.CTRL_BREAK_EVENT # type: ignore[attr-defined] else: From f9010ea14d19e974f091731e4677a0f2eaee45d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 May 2025 09:48:21 -0500 Subject: [PATCH 28/39] chore(deps): update pytest-timeout requirement from ==2.3.* to ==2.4.* (#192) Updates the requirements on [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) to permit the latest version. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.3.0...2.4.0) --- updated-dependencies: - dependency-name: pytest-timeout dependency-version: 2.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index b1fda23e..7d0f6623 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -2,7 +2,7 @@ coverage[toml] == 7.* coverage-conditional-plugin == 0.9.0 pytest == 7.4.* pytest-cov == 6.1.* -pytest-timeout == 2.3.* +pytest-timeout == 2.4.* pytest-xdist == 3.6.* black == 25.* ruff == 0.11.* From 9fde71892da7fa94b112145aba6c5081d62fe572 Mon Sep 17 00:00:00 2001 From: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Date: Wed, 14 May 2025 13:11:06 -0500 Subject: [PATCH 29/39] chore(deps): update pytest to 8.3.* (#193) Signed-off-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- .../integ/background/test_background_mode.py | 5 +++++ .../adaptor_runtime/integ/test_integration_entrypoint.py | 7 ++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 7d0f6623..9821a776 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,6 +1,6 @@ coverage[toml] == 7.* coverage-conditional-plugin == 0.9.0 -pytest == 7.4.* +pytest == 8.3.* pytest-cov == 6.1.* pytest-timeout == 2.4.* pytest-xdist == 3.6.* diff --git a/test/openjd/adaptor_runtime/integ/background/test_background_mode.py b/test/openjd/adaptor_runtime/integ/background/test_background_mode.py index 5357f482..1fdc16ca 100644 --- a/test/openjd/adaptor_runtime/integ/background/test_background_mode.py +++ b/test/openjd/adaptor_runtime/integ/background/test_background_mode.py @@ -29,10 +29,15 @@ mod_path = (Path(__file__).parent.parent).resolve() sys.path.append(str(mod_path)) + if (_pypath := os.environ.get("PYTHONPATH")) is not None: os.environ["PYTHONPATH"] = os.pathsep.join((_pypath, str(mod_path))) else: os.environ["PYTHONPATH"] = str(mod_path) + +# Add the module path to PYTHONPATH for subprocesses +os.environ["PYTEST_XDIST_WORKER_PYTHONPATH"] = str(mod_path) + from AdaptorExample import AdaptorExample # noqa: E402 diff --git a/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py b/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py index 0218c7e4..22b0d5e3 100644 --- a/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py +++ b/test/openjd/adaptor_runtime/integ/test_integration_entrypoint.py @@ -20,10 +20,15 @@ mod_path = Path(__file__).parent.resolve() sys.path.append(str(mod_path)) + if (_pypath := os.environ.get("PYTHONPATH")) is not None: - os.environ["PYTHONPATH"] = ":".join((_pypath, str(mod_path))) + os.environ["PYTHONPATH"] = os.pathsep.join([_pypath, str(mod_path)]) else: os.environ["PYTHONPATH"] = str(mod_path) + +# Add the module path to PYTHONPATH for subprocesses +os.environ["PYTEST_XDIST_WORKER_PYTHONPATH"] = str(mod_path) + from CommandAdaptorExample import CommandAdaptorExample # noqa: E402 From 26877c71c7948f77dba64435f5601bb60e9ffdcd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 21:23:01 +0000 Subject: [PATCH 30/39] chore(deps): update hatch-vcs requirement from ==0.4.* to ==0.5.* Updates the requirements on [hatch-vcs](https://github.com/ofek/hatch-vcs) to permit the latest version. - [Release notes](https://github.com/ofek/hatch-vcs/releases) - [Changelog](https://github.com/ofek/hatch-vcs/blob/master/HISTORY.md) - [Commits](https://github.com/ofek/hatch-vcs/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: hatch-vcs dependency-version: 0.5.0 dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: Justin Blagden --- requirements-development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-development.txt b/requirements-development.txt index 5997871b..635448e7 100644 --- a/requirements-development.txt +++ b/requirements-development.txt @@ -1,2 +1,2 @@ hatch == 1.14.* -hatch-vcs == 0.4.* \ No newline at end of file +hatch-vcs == 0.5.* \ No newline at end of file From ac63a2edf248251ec45e930b0e9374c8b4de9cab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 17:57:05 +0000 Subject: [PATCH 31/39] chore(deps): update pytest-xdist requirement from ==3.6.* to ==3.7.* Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.6.0...v3.7.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-version: 3.7.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 9821a776..453e12f8 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -3,7 +3,7 @@ coverage-conditional-plugin == 0.9.0 pytest == 8.3.* pytest-cov == 6.1.* pytest-timeout == 2.4.* -pytest-xdist == 3.6.* +pytest-xdist == 3.7.* black == 25.* ruff == 0.11.* mypy == 1.15.* From 4990ab33164d441edde204fb3047cb910f146e1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 19:02:58 +0000 Subject: [PATCH 32/39] chore(deps): update pytest requirement from ==8.3.* to ==8.4.* Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.0.dev0...8.4.0) --- updated-dependencies: - dependency-name: pytest dependency-version: 8.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 453e12f8..c0f195e9 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,6 +1,6 @@ coverage[toml] == 7.* coverage-conditional-plugin == 0.9.0 -pytest == 8.3.* +pytest == 8.4.* pytest-cov == 6.1.* pytest-timeout == 2.4.* pytest-xdist == 3.7.* From c865716c042ba0c19266a1d6b0b363adb3cfba55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Jun 2025 01:35:20 +0000 Subject: [PATCH 33/39] chore(deps): update mypy requirement from ==1.15.* to ==1.16.* Updates the requirements on [mypy](https://github.com/python/mypy) to permit the latest version. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.15.0...v1.16.0) --- updated-dependencies: - dependency-name: mypy dependency-version: 1.16.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Justin Blagden --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index c0f195e9..bbc35dc4 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -6,6 +6,6 @@ pytest-timeout == 2.4.* pytest-xdist == 3.7.* black == 25.* ruff == 0.11.* -mypy == 1.15.* +mypy == 1.16.* psutil == 7.0.* types-PyYAML ~= 6.0 From 4a4d794d29cb9542964945d62550a0d204b0a36a Mon Sep 17 00:00:00 2001 From: Morgan Epp <60796713+epmog@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:10:19 -0500 Subject: [PATCH 34/39] fix: sdist failed to install (#199) Signed-off-by: Morgan Epp <60796713+epmog@users.noreply.github.com> Signed-off-by: Justin Blagden --- pyproject.toml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2c25f359..4b4d2727 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,11 +66,9 @@ destinations = [ ] [tool.hatch.build.targets.sdist] -packages = [ - "src/openjd", -] -only-include = [ +include = [ "src/openjd", + "hatch_version_hook.py", ] [tool.hatch.build.targets.wheel] From 46be6cb6a779e54a7fe10fe1f4fcdc83765d7ce8 Mon Sep 17 00:00:00 2001 From: Mark <399551+mwiebe@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:37:32 -0700 Subject: [PATCH 35/39] docs: Refine the introduction, move usage into docs dir * Changes to the introduction are to clarify the purpose of this library, making it easier to understand what features it provides. * Moved the content about using the library from README.md into a docs directory. With the re-organization, there's a place to add new content like worked examples or more details about interface features the library supports. Signed-off-by: Mark <399551+mwiebe@users.noreply.github.com> Signed-off-by: Justin Blagden --- README.md | 172 ++++++++++++------------------------------------- docs/README.md | 103 +++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 130 deletions(-) create mode 100644 docs/README.md diff --git a/README.md b/README.md index 0d352919..d29e9368 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,47 @@ -# Open Job Description - Adaptor Runtime +# Open Job Description - Adaptor Runtime Library [![pypi](https://img.shields.io/pypi/v/openjd-adaptor-runtime.svg?style=flat)](https://pypi.python.org/pypi/openjd-adaptor-runtime) [![python](https://img.shields.io/pypi/pyversions/openjd-adaptor-runtime.svg?style=flat)](https://pypi.python.org/pypi/openjd-adaptor-runtime) [![license](https://img.shields.io/pypi/l/openjd-adaptor-runtime.svg?style=flat)](https://github.com/OpenJobDescription/openjd-adaptor-runtime/blob/mainline/LICENSE) -This package provides a runtime library for creating a command-line Adaptor to assist with -integrating an existing application, such as a rendering application, into batch computing systems -that run Jobs in a way that is compatible with [Open Job Description Sessions]. That is, when running -a Job on a host consists of a phase to initialize a local compute environment, then running one -or more Tasks that each run the same application for the Job on the host, and finally tearing down -the initialized compute environment when complete. - -Some of the reasons that you should consider creating an Adaptor are if you want to: - -1. Optimize the runtime of your Job on a compute host by loading the application once and dynamically running - many units of work with that single application instance before shutting down the application; -2. Programmatically respond to signals that the application provides, such as stopping the application early - if it prints a message to stdout that indicates that the run may produce undesirable results like watermarks - due to missing a floating license, or a bad image render due to missing textures; -3. Dynamically select which version of an application to run based on what is available and modify the - command-line options provided to the application based on which version will be run; -4. Emit [Open Job Description Stdout Messages], or equivalent, to update the batch computing system on the - status or progress of the running unit of work; and/or -5. Integrate [Open Job Description Path Mapping] information into the application in the format that it is expecting. +This package provides a runtime library to help build application interfaces that simplify +Open Job Description job templates. When implemented by a third party on behalf of +an application, the result is a CLI command that acts as an adaptor. Application +developers can also implement support for these CLI patterns directly in their +applications, potentially using this library to simplify the work. + +Interface features that this library can assist with include: + +1. Run as a background daemon to amortize application startup and scene load time. + * Tasks run in the context of [Open Job Description Sessions], and this pattern lets a + scheduling engine sequentially dispatch tasks to a single process that retains the + application, loaded scene, and any acceleration data structures in memory. +2. Report progress and status messages. + * Applications write progress information and status messages in many different ways. + An adaptor can scan the output of an application and report it in the format specified + for [Open Job Description Stdout Messages]. +3. Map file system paths in input data. + * When running tasks on a different operating system, or when files are located at + different locations compared to where they were at creation, an adaptor can take + path mapping rules and perform [Open Job Description Path Mapping]. +4. Transform signals like cancelation requests from the Open Job Description runtime into + the signal needed by the application. + * Applications may require different mechanisms to receive these messages, an adaptor + can handle any differences with what Open Job Description provides to give full + feature support. +5. Adjust application default behaviors for batch processing. + * When running applications that were built for interactive use within a batch processing + system, some default behaviors may lead to unreliability of workload completion, such + as using watermarks when a license could not be acquired or returning a success exit + code when an input data file could not be read. The adaptor can monitor and detect + these cases. [Open Job Description Sessions]: https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run#sessions [Open Job Description Stdout Messages]: https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run#stdoutstderr-messages [Open Job Description Path Mapping]: https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run#path-mapping +Read the [Library Documentation](docs/README.md) to learn more. + ## Compatibility This library requires: @@ -37,13 +51,13 @@ This library requires: ## Versioning -This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its +This package's version follows [Semantic Versioning 2.0](https://semver.org/), but is still considered to be in its initial development, thus backwards incompatible versions are denoted by minor version bumps. To help illustrate how versions will increment during this initial development stage, they are described below: -1. The MAJOR version is currently 0, indicating initial development. -2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. -3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. +1. The MAJOR version is currently 0, indicating initial development. +2. The MINOR version is currently incremented when backwards incompatible changes are introduced to the public API. +3. The PATCH version is currently incremented when bug fixes or backwards compatible changes are introduced to the public API. ## Downloading @@ -55,115 +69,13 @@ You can download this package from: See [VERIFYING_PGP_SIGNATURE](VERIFYING_PGP_SIGNATURE.md) for more information. -## Usage - -To create your own Adaptor you create a Python application that uses this package and consists of: - -1. A console script entrypoint that passes control flow to an instance of this runtime's **EntryPoint** class; -2. A JSON file for configuration options of your Adaptor; and -3. An Adaptor class derived from either the **CommandAdaptor** or **Adaptor** class. - - **CommandAdaptor** is ideal for applications where you do not need to initialize the local compute environment by, say, - preloading your application, and simply need to run a single commandline for each Task that is run on the compute host. - Please see [CommandAdaptorExample] in this GitHub repository for a simple example. - - **Adaptor** exposes callbacks for every stage of an Adaptor's lifecycle, and is is suited for Adaptors where you want full control. - Please see [AdaptorExample] in this GitHub repository for a simple example. - -You can also find many more examples within the [AWS Deadline Cloud Organization] on GitHub. - -[CommandAdaptorExample]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/tree/release/test/openjd/adaptor_runtime/integ/CommandAdaptorExample -[AdaptorExample]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/tree/mainline/test/openjd/adaptor_runtime/integ/AdaptorExample -[AWS Deadline Cloud Organization]: https://github.com/aws-deadline - -### Adaptor Lifecycle - -All Adaptors undergo a lifecycle consisting of the following stages: - -1. **start**: Occurs once during construction and initialization of the Adaptor. This is the stage where - your Adaptor should perform any expensive initialization actions for the local compute environment; such - as starting and loading an application in the background for use in later stages of the Adaptor's lifecycle. - - Runs the `on_start()` method of Adaptors derived from the **Adaptor** base class. -2. **run**: May occur one or more times for a single running Adaptor. This is the stage where your Adaptor is - performing the work required of a Task that is being run. - - Run the `on_run()` method of Adaptors derived from the **Adaptor** base class. - - Run the `on_prerun()` then `get_managed_process()` then `on_postrun()` methods of Adaptors derived from the - **CommandAdaptor** base class. -3. **stop**: Occurs once as part of shutting down the Adaptor. This stage is the inverse of the **start** - stage and should undo the actions done in that phase; such as stopping any background processes that are - still running. - - Runs the `on_stop()` method of Adaptors derived from the **Adaptor** base class. -4. **cleanup**: A final opportunity to cleanup any remaining processes and data left behind by the Adaptor. - - Runs the `on_cleanup()` method of Adaptors derived from the **Adaptor** base class. - -A running Adaptor can also be canceled by sending the Adaptor process a signal (SIGINT/SIGTERM on posix, or -CTRL-C/CTRL-BREAK on Windows). This will call the `on_cancel()` method of your Adaptor, if one is defined. -You should ensure that the design of your Adaptor allows this cancelation to interrupt any actions that may -be running, and gracefully exit any running background processes. - -### Running an Adaptor - -The **EntryPoint** provided by this runtime allows for an Adaptor to be run directly through its -entire lifecycle in a single command, or to be run as a background daemon that lets you drive the lifecycle -of the Adaptor yourself. - -#### The `run` Subcommand - -The `run` subcommand of an Adaptor will run it through its entire lifecycle (**start**, then **run**, then -**stop**, and finally **cleanup**), and then exit. This is useful for initial development and testing, and -for running Adaptors created from the **CommandAdaptor** base class. - -To see this in action install the openjd-adaptor-runtime package into your Python environment, and then -within your local clone of this repository: - -```bash -cd test/openjd -python3 -m integ.AdaptorExample run --init-data '{"name": "MyAdaptor"}' --run-data '{"hello": "world"}' -``` - -The arguments to the `run` subcommand are: - -- **`--init-data`** is a JSON-encoded dictionary either inline or in a given file (`file://`). This data is - decoded and automatically stored in the `self.init_data` member of the running Adaptor. -- **`--run-data`** is, similarly, a JSON-encoded dictionary either inline or in a given file (`file://`). - This data is passed as the argument to the `on_run()` method of an **Adaptor** or the `get_managed_process()` - method of a **CommandAdaptor**. - -#### The `daemon` Subcommand - -With the `daemon` subcommand, you must transition the Adaptor through its lifecycle yourself by running the -subcommands of the `daemon` subcommand in order. - -1. Start the Adaptor: Initializes the Adaptor as a background daemon subprocess and leaves it running. - This runs the `on_start()` method of your **Adaptor**-derived Adaptor if the method is available. - ``` - python -m integ.AdaptorExample daemon start --connection-file ./AdaptorExampleConnection.json --init-data '{"name": "MyAdaptor"}' - ``` - - **`--init-data`** is as described in the `run` subcommand, above. - - **`--connection-file`** provide a path to a JSON file for the Adaptor to create. This file contains information - on how to connect to the daemon subprocess remains running, and you must provide it to all subsequent runs of the - Adaptor until you have stopped it. -2. Run the Adaptor: Connects to the daemon subprocess that is running the Adaptor and instructs it to perform its **run** - lifecycle phase. The command remains connected to the daemon subprocess for the entire duration of this **run** phase, - and forwards all data logged by the Adaptor to stdout or stderr. This step can be repeated multiple times. - ``` - python -m integ.AdaptorExample daemon run --connection-file ./AdaptorExampleConnection.json --run-data '{"hello": "world"}' - ``` - - **`--run-data`** is as described in the `run` subcommand, above. - - **`--connection-file`** is as described in above. -3. Stop the Adaptor: Connects to the daemon subprocess that is running the Adaptor and instructs it to transition to the - **stop** then **cleanup** lifecycle phases, and then instructs the daemon subprocess to exit when complete. The command - remains connected to the daemon subprocess for the entire duration, and forwards all data logged by the Adaptor to stdout - or stderr. - ``` - python -m integ.AdaptorExample daemon stop --connection-file ./AdaptorExampleConnection.json - ``` - ## Security -We take all security reports seriously. When we receive such reports, we will -investigate and subsequently address any potential vulnerabilities as quickly -as possible. If you discover a potential security issue in this project, please +We take all security reports seriously. When we receive such reports, we will +investigate and subsequently address any potential vulnerabilities as quickly +as possible. If you discover a potential security issue in this project, please notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) -or directly via email to [AWS Security](aws-security@amazon.com). Please do not +or directly via email to [AWS Security](aws-security@amazon.com). Please do not create a public GitHub issue in this project. ## License diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..ebafc038 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,103 @@ +# Open Job Description Adaptor Runtime Library - Documentation + +## Usage + +To create your own Adaptor you create a Python application that uses this package and consists of: + +1. A console script entrypoint that passes control flow to an instance of this runtime's **EntryPoint** class; +2. A JSON file for configuration options of your Adaptor; and +3. An Adaptor class derived from either the **CommandAdaptor** or **Adaptor** class. + - **CommandAdaptor** is ideal for applications where you do not need to initialize the local compute environment by, say, + preloading your application, and simply need to run a single commandline for each Task that is run on the compute host. + Please see [CommandAdaptorExample] in this GitHub repository for a simple example. + - **Adaptor** exposes callbacks for every stage of an Adaptor's lifecycle, and is is suited for Adaptors where you want full control. + Please see [AdaptorExample] in this GitHub repository for a simple example. + +You can also find many more examples within the [AWS Deadline Cloud Organization] on GitHub. + +[CommandAdaptorExample]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/tree/release/test/openjd/adaptor_runtime/integ/CommandAdaptorExample +[AdaptorExample]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/tree/mainline/test/openjd/adaptor_runtime/integ/AdaptorExample +[AWS Deadline Cloud Organization]: https://github.com/aws-deadline + +### Adaptor Lifecycle + +All Adaptors undergo a lifecycle consisting of the following stages: + +1. **start**: Occurs once during construction and initialization of the Adaptor. This is the stage where + your Adaptor should perform any expensive initialization actions for the local compute environment; such + as starting and loading an application in the background for use in later stages of the Adaptor's lifecycle. + - Runs the `on_start()` method of Adaptors derived from the **Adaptor** base class. +2. **run**: May occur one or more times for a single running Adaptor. This is the stage where your Adaptor is + performing the work required of a Task that is being run. + - Run the `on_run()` method of Adaptors derived from the **Adaptor** base class. + - Run the `on_prerun()` then `get_managed_process()` then `on_postrun()` methods of Adaptors derived from the + **CommandAdaptor** base class. +3. **stop**: Occurs once as part of shutting down the Adaptor. This stage is the inverse of the **start** + stage and should undo the actions done in that phase; such as stopping any background processes that are + still running. + - Runs the `on_stop()` method of Adaptors derived from the **Adaptor** base class. +4. **cleanup**: A final opportunity to cleanup any remaining processes and data left behind by the Adaptor. + - Runs the `on_cleanup()` method of Adaptors derived from the **Adaptor** base class. + +A running Adaptor can also be canceled by sending the Adaptor process a signal (SIGINT/SIGTERM on posix, or +CTRL-C/CTRL-BREAK on Windows). This will call the `on_cancel()` method of your Adaptor, if one is defined. +You should ensure that the design of your Adaptor allows this cancelation to interrupt any actions that may +be running, and gracefully exit any running background processes. + +### Running an Adaptor + +The **EntryPoint** provided by this runtime allows for an Adaptor to be run directly through its +entire lifecycle in a single command, or to be run as a background daemon that lets you drive the lifecycle +of the Adaptor yourself. + +#### The `run` Subcommand + +The `run` subcommand of an Adaptor will run it through its entire lifecycle (**start**, then **run**, then +**stop**, and finally **cleanup**), and then exit. This is useful for initial development and testing, and +for running Adaptors created from the **CommandAdaptor** base class. + +To see this in action install the openjd-adaptor-runtime package into your Python environment, and then +within your local clone of this repository: + +```bash +cd test/openjd +python3 -m integ.AdaptorExample run --init-data '{"name": "MyAdaptor"}' --run-data '{"hello": "world"}' +``` + +The arguments to the `run` subcommand are: + +- **`--init-data`** is a JSON-encoded dictionary either inline or in a given file (`file://`). This data is + decoded and automatically stored in the `self.init_data` member of the running Adaptor. +- **`--run-data`** is, similarly, a JSON-encoded dictionary either inline or in a given file (`file://`). + This data is passed as the argument to the `on_run()` method of an **Adaptor** or the `get_managed_process()` + method of a **CommandAdaptor**. + +#### The `daemon` Subcommand + +With the `daemon` subcommand, you must transition the Adaptor through its lifecycle yourself by running the +subcommands of the `daemon` subcommand in order. + +1. Start the Adaptor: Initializes the Adaptor as a background daemon subprocess and leaves it running. + This runs the `on_start()` method of your **Adaptor**-derived Adaptor if the method is available. + ``` + python -m integ.AdaptorExample daemon start --connection-file ./AdaptorExampleConnection.json --init-data '{"name": "MyAdaptor"}' + ``` + - **`--init-data`** is as described in the `run` subcommand, above. + - **`--connection-file`** provide a path to a JSON file for the Adaptor to create. This file contains information + on how to connect to the daemon subprocess remains running, and you must provide it to all subsequent runs of the + Adaptor until you have stopped it. +2. Run the Adaptor: Connects to the daemon subprocess that is running the Adaptor and instructs it to perform its **run** + lifecycle phase. The command remains connected to the daemon subprocess for the entire duration of this **run** phase, + and forwards all data logged by the Adaptor to stdout or stderr. This step can be repeated multiple times. + ``` + python -m integ.AdaptorExample daemon run --connection-file ./AdaptorExampleConnection.json --run-data '{"hello": "world"}' + ``` + - **`--run-data`** is as described in the `run` subcommand, above. + - **`--connection-file`** is as described above. +3. Stop the Adaptor: Connects to the daemon subprocess that is running the Adaptor and instructs it to transition to the + **stop** then **cleanup** lifecycle phases, and then instructs the daemon subprocess to exit when complete. The command + remains connected to the daemon subprocess for the entire duration, and forwards all data logged by the Adaptor to stdout + or stderr. + ``` + python -m integ.AdaptorExample daemon stop --connection-file ./AdaptorExampleConnection.json + ``` From f3ccf6f134abba1d184d3aa8ede5f29026ee3475 Mon Sep 17 00:00:00 2001 From: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Date: Thu, 12 Jun 2025 09:02:48 -0500 Subject: [PATCH 36/39] ci: update to tag based release workflows (#201) Signed-off-by: Charles Moore <122481442+moorec-aws@users.noreply.github.com> Signed-off-by: Justin Blagden --- .github/workflows/code_quality.yml | 4 +++ .github/workflows/release_bump.yml | 7 ---- .github/workflows/release_publish.yml | 52 ++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index b6600046..e6ec9ee1 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -8,6 +8,9 @@ on: branch: required: false type: string + tag: + required: false + type: string jobs: Test: @@ -20,3 +23,4 @@ jobs: with: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} + ref: ${{inputs.tag}} diff --git a/.github/workflows/release_bump.yml b/.github/workflows/release_bump.yml index d079a049..6b6a87dd 100644 --- a/.github/workflows/release_bump.yml +++ b/.github/workflows/release_bump.yml @@ -17,15 +17,8 @@ concurrency: group: release jobs: - UnitTests: - name: Unit Tests - uses: ./.github/workflows/code_quality.yml - with: - branch: mainline - Bump: name: Version Bump - needs: UnitTests uses: OpenJobDescription/.github/.github/workflows/reusable_bump.yml@mainline secrets: inherit with: diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 788e7c31..9a0570d9 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -1,5 +1,5 @@ name: "Release: Publish" -run-name: "Release: ${{ github.event.head_commit.message }}" +run-name: "Release: ${{ github.event.head_commit.message || inputs.tag }}" on: push: @@ -7,18 +7,62 @@ on: - mainline paths: - CHANGELOG.md + workflow_dispatch: + inputs: + tag: + required: true + type: string + description: Specify a tag to re-run a release. concurrency: group: release +permissions: + contents: read + jobs: + TagRelease: + uses: OpenJobDescription/.github/.github/workflows/reusable_tag_release.yml@mainline + secrets: inherit + with: + tag: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || '' }} + + UnitTests: + name: Unit Tests + needs: TagRelease + uses: ./.github/workflows/code_quality.yml + with: + tag: ${{ needs.TagRelease.outputs.tag }} + + PreRelease: + needs: [TagRelease, UnitTests] + uses: OpenJobDescription/.github/.github/workflows/reusable_prerelease.yml@mainline + permissions: + id-token: write + contents: write + secrets: inherit + with: + tag: ${{ needs.TagRelease.outputs.tag }} + + Release: + needs: [TagRelease, PreRelease] + uses: OpenJobDescription/.github/.github/workflows/reusable_release.yml@mainline + secrets: inherit + permissions: + id-token: write + contents: write + with: + tag: ${{ needs.TagRelease.outputs.tag }} + Publish: - name: Publish Release + needs: [TagRelease, Release] + uses: OpenJobDescription/.github/.github/workflows/reusable_publish_python.yml@mainline permissions: id-token: write - contents: write - uses: OpenJobDescription/.github/.github/workflows/reusable_publish.yml@mainline secrets: inherit + with: + tag: ${{ needs.TagRelease.outputs.tag }} + # PyPI does not support reusable workflows yet # # See https://github.com/pypi/warehouse/issues/11096 PublishToPyPI: From 0068076cd4a1c6bc990e3540a55c275ffa386506 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Thu, 12 Jun 2025 08:33:32 -0700 Subject: [PATCH 37/39] docs: Write a worked example of defining a Blender adaptor (#202) This is a proposal to try a very different adaptor interface than is currently used. Signed-off-by: Mark <399551+mwiebe@users.noreply.github.com> Signed-off-by: Justin Blagden --- docs/README.md | 4 + docs/blender_worked_example.md | 485 +++++++++++++++++++++++++++++++++ 2 files changed, 489 insertions(+) create mode 100644 docs/blender_worked_example.md diff --git a/docs/README.md b/docs/README.md index ebafc038..769795b2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,10 @@ You can also find many more examples within the [AWS Deadline Cloud Organization [AdaptorExample]: https://github.com/OpenJobDescription/openjd-adaptor-runtime-for-python/tree/mainline/test/openjd/adaptor_runtime/integ/AdaptorExample [AWS Deadline Cloud Organization]: https://github.com/aws-deadline +We have some ideas for better ways to structure adaptors. See +[this Blender worked example](blender_worked_example.md) +for a proposed new application interface for Blender. + ### Adaptor Lifecycle All Adaptors undergo a lifecycle consisting of the following stages: diff --git a/docs/blender_worked_example.md b/docs/blender_worked_example.md new file mode 100644 index 00000000..4f76b9bd --- /dev/null +++ b/docs/blender_worked_example.md @@ -0,0 +1,485 @@ +# PROPOSAL: Worked Example of a Blender Adaptor + +## Table of Contents + +- [Introduction](#introduction) +- [Batch Workloads in Blender](#batch-workloads-in-blender) + - [Rendering](#rendering) + - [Custom Scripting](#custom-scripting) +- [Evaluate Open Job Description application interface patterns](#evaluate-open-job-description-application-interface-patterns) + - [Run as a background daemon](#run-as-a-background-daemon) + - [Report progress and status messages](#report-progress-and-status-messages) + - [Map file system paths](#map-file-system-paths) + - [Task chunking](#task-chunking) +- [Designing the Blender adaptor CLI interface](#designing-the-blender-adaptor-cli-interface) + - [Requirements for the adaptor user experience](#requirements-for-the-adaptor-user-experience) + - [Adaptor CLI interface design](#adaptor-cli-interface-design) + - [Map file system paths](#map-file-system-paths-1) + - [Render task chunks](#render-task-chunks) + +## Introduction + +This document provides a worked example of the thought process behind designing a new interface for +the Blender Open Job Description adaptor, implemented in +[deadline-cloud-for-blender](https://github.com/aws-deadline/deadline-cloud-for-blender). +The current adaptor interface is significantly different from how users run Blender batch +workloads today, and I believe we can do better. + +The goal is to build an application interface for Blender that simplifies Open Job Description job templates for Blender workloads. +This means the adaptor should be intuitive for someone already familiar with running workloads in Blender, but also have +simple ways to support the features listed in the [project README](../README.md). + +Therefore, before we can define this adaptor, we must understand Blender's existing support +for batch workloads. Then we can identify what benefits an adaptor would provide over just +using Blender's existing support, and finally design the application interface it should +provide. + +If this is successful, we propose to extend this same thought process to all the other `deadline-cloud-for-` +projects, and modify the [job_env_daemon_process](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/job_bundles/job_env_daemon_process/template.yaml) +sample job template to demonstrate this library as well. + +## Batch Workloads in Blender + +Our first step is to understand the batch workloads users want to run in Blender. +The Blender documentation topic [Using Blender From The Command Line](https://docs.blender.org/manual/en/latest/advanced/command_line/index.html) +describes two cases: [rendering animation](https://docs.blender.org/manual/en/latest/advanced/command_line/render.html#command-line-render), +and launching Blender with [different arguments](https://docs.blender.org/manual/en/latest/advanced/command_line/arguments.html#command-line-args). + +### Rendering + +Two options, `-f` (`--render-frame`) and `-a` (`--render-anim`), cause it to render frames and save the images. +Here's an example rendering command you could include in a job template: + +```sh +blender --background '{{Param.BlenderSceneFile}}' \ + --render-output '{{Param.OutputDir}}/{{Param.OutputPattern}}' \ + --render-format {{Param.Format}} \ + --use-extension 1 \ + --render-frame {{Task.Param.Frame}} +``` + +### Custom Scripting + +The options `-P` (`--python`), `--python-text`, and `--python-expr` cause it to run Python code. +Here's an example custom workload command using the Blender Python API for file conversion: + +```sh +blender --factory-startup \ + --background \ + --python-exit-code 1 \ + --python-use-system-env \ + --python scripts/blender_file_conversion.py \ + -- \ + --input {{Param.InputFile}} \ + --output {{Param.OutputFile}} +``` + +Inside `blender_file_conversion.py` you might find the following argument processing: + +```python +import argparse +... + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args(argv) + + run_file_conversion(args.input, args.output) + +if __name__ == "__main__": + main(sys.argv[sys.argv.index("--") + 1]) +``` + +## Evaluate Open Job Description application interface patterns + +With an idea of the batch workloads to run on Blender, let's look at how they fit into +Open Job Description job templates expressing various workload patterns. + +### Run as a background daemon + +For this pattern, Blender should stay open in the background, then the job runs commands +in sequence to render individual frames or run custom scripts in that persistent Blender +process. The Deadline Cloud samples github repository has an example job template, +[Job Environment Daemon Process](https://github.com/aws-deadline/deadline-cloud-samples/blob/mainline/job_bundles/job_env_daemon_process/template.yaml) +to illustrate the pattern where a step environment called DaemonProcess loads a process in +the background for the tasks of the step EnvWithDaemonProcess. + +The [Blender CLI command](https://docs.blender.org/manual/en/latest/advanced/command_line/arguments.html) +has a `--background` option, but this option means to not show a GUI, not to load a +persistent background daemon and then send commands to it. + +As there is no built-in background daemon support in Blender, but Blender provides +flexible Python scripting, we can add support for it in an adaptor. + +### Report progress and status messages + +While rendering, Blender prints output that look like this: + +```sh +$ blender --background '/.../blender-3.5-splash.blend' \ + --render-output '/.../output_####' \ + --render-format JPEG \ + --use-extension 1 \ + --render-frame 1 +Blender 4.4.0 (hash 05377985c527 built 2025-03-18 03:01:40) +Read blend: "/.../blender-3.5-splash.blend" +Fra:1 Mem:421.62M (Peak 422.43M) | Time:00:12.91 | Mem:0.00M, Peak:0.00M | Main, ViewLayer | Synchronizing object | Icosphere +... +Fra:1 Mem:564.29M (Peak 663.39M) | Time:00:15.28 | Mem:337.70M, Peak:340.61M | Main, ViewLayer | Loading denoising kernels (may take a few minutes the first time) +Fra:1 Mem:564.29M (Peak 663.39M) | Time:00:15.28 | Mem:337.70M, Peak:340.61M | Main, ViewLayer | Sample 0/128 +Fra:1 Mem:684.53M (Peak 684.53M) | Time:00:23.55 | Remaining:17:31.44 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Sample 1/128 +... +Fra:1 Mem:735.16M (Peak 811.09M) | Time:17:27.41 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Sample 128/128 +Fra:1 Mem:735.16M (Peak 811.09M) | Time:17:27.41 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Finished +Fra:1 Mem:292.20M (Peak 811.09M) | Time:17:27.44 | Compositing +Saved: '/.../output_0001.jpg' +Time: 17:27.67 (Saving: 00:00.17) +``` + +Per the [Open Job Description stdout/stderr messages documentation](https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run#stdoutstderr-messages), +Blender could print message like the following to report completion progress and status messages, that the queuing +system will know to report back to the user in their dashboard view: + +``` +openjd_progress: 32% +openjd_status: Loading denoising kernels (may take a few minutes the first time) +``` + +As Blender does not have an option to print messages in this format, we can add support in an adaptor, by reading +the output from Blender and printing lines in the Open Job Description format. + +### Map file system paths + +When a queuing system runs jobs on a different operating system, or with files located at different paths, +the job needs to transform any absolute path references in the input data for processing to work. +Open Job Description provides [Path Mapping rules](https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run#path-mapping) +that a job template can use to learn about which paths to map. + +Blender has options to control use of [Relative vs Absolute Paths](https://docs.blender.org/manual/en/latest/files/blend/open_save.html#files-blend-relative-paths), +and defaults to using relative paths when possible. If a Blender scene uses only relative paths, no +custom path mapping will be necessary, because the system will handle re-mapping the scene file path +as a job parameter with type PATH, and all the other paths will be relative to that. Users can choose +to use absolute paths, or on Windows they can use multiple drive letters, in which case there is no +way to make all paths relative. In those cases, the job template will need to handle path mapping. + +As Blender does not have a built-in path mapping mechanism based on the Open Job Description path +mapping rule format, this is something for the adaptor to handle. + +### Task chunking + +Open Job Description [RFC 0001](https://github.com/OpenJobDescription/openjd-specifications/blob/mainline/rfcs/0001-task-chunking.md) +introduced a mechanism to run multiple tasks in a single action as a chunk. When running a chunk, the frame +numbers are provided as an [Open Job Description integer range expression](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#34111-intrangeexpr). +The Blender `--render-frame` CLI option supports a syntax that is similar, but different from this syntax. +An adaptor can accept the Open Job Description syntax and transform it for Blender. + +## Designing the Blender adaptor CLI interface + +### Requirements for the adaptor user experience + +The best user experience would be for users to directly use the Blender CLI in Open Job Description +job templates, and for it to be simple and straightforward. This leads us to our first goal of the +CLI interface we will design: + +* The adaptor CLI interface should be as close to the existing Blender CLI as we can make it, with + the following properties: + * If an existing Blender CLI option is useful for batch processing, the adaptor should support it exactly. + * If the adaptor includes a CLI option that is not part of the Blender CLI, try to make it fit + in to the Blender CLI commands as naturally as possible. To evaluate an addition, imagine pitching to + the Blender Foundation that they should add the command to Blender directly, and consider how convincing + your case is. + +Because the adaptor will be a separate command from Blender like `blender-openjd` instead of `blender`, +authors of job templates should be able to easily take a CLI `blender` command, switch it to `blender-openjd`, +and make minimal changes to use the Open Job Description features. Similarly, taking a `blender-openjd` command +and adjusting it to be a straight `blender` command should also be easy. + +* Any adaptor CLI options that are not in the `blender` CLI should be easily identifiable. + * If the feature is specific to Open Job Description, likely include `openjd` in the option name somewhere. + * If the feature could be added to the `blender` CLI directly without any Open Job Description + reference, and would make sense, craft a clear rationale for the choice. + +With the desire to keep the possibility of Blender itself implementing the CLI options of the adaptor CLI, +the behavior of the adaptor needs to be compatible. That leads to: + +* When calling the adaptor CLI with options also available in the Blender CLI, the behavior such as the + output it prints should be as identical to Blender as possible. + +### Adaptor CLI interface design + +#### Start a background Blender daemon + +Starting a daemon returns an address necessary to access it later, that can be stored in an environment +variable. This address is used by the run and stop daemon operations. Most workloads will start only one +Blender daemon, and we will provide a simple way to express this in a job template without boilerplate. +Some workloads will start multiple Blender daemons, and they must track multiple daemon addresses. + +We will add two CLI options for starting a background Blender daemon. The `--background` cannot be used together with +these options, but it is automatically enabled when they are used. Additional options provided in the command are the same as for the +`blender` command, and are directives for how to initialize Blender once it has loaded, such as to open a `.blend` scene file, +apply path mapping rules, or modify the output image resolution. + +```sh +# Start a Blender background daemon and print 'openjd_env: BLENDER_DAEMON_ADDRESS=' +# for Open Job Description to use. +$ blender-openjd --openjd-daemon-start ... +openjd_env: BLENDER_DAEMON_ADDRESS= + +# Start a Blender background daemon and print ''. Can be used from any shell +# scripting context like 'export BLENDER_DAEMON_ADDRESS=$(blender-openjd --daemon-start ...)' +$ blender-openjd --daemon-start ... + +``` + +Here is example usage within a job template +[onEnter](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#43-environmentactions) +action. + +```yaml +... + actions: + onEnter: + # This form lets you directly start the daemon without manipulating any variables + command: blender-openjd + args: ["--openjd-daemon-start", "{{Param.BlenderFile}}"] +``` + +```yaml +... + actions: + onEnter: + command: "bash" + args: ["{{Env.File.OnEnter}}"] + embeddedFiles: + - name: OnEnter + filename: on-enter.sh + type: TEXT + data: | + set -euo pipefail + + # This matches the default behavior of the --openjd-daemon-start option + export BLENDER_DAEMON_ADDRESS="$(blender-openjd --daemon-start {{Param.BlenderFile}})" + echo "openjd_env: BLENDER_DAEMON_ADDRESS=$BLENDER_DAEMON_ADDRESS" + + # You can use this form to start multiple independent Blender background daemons. + export BLENDER_DAEMON_ADDRESS_2="$(blender-openjd --daemon-start {{Param.BlenderFile}})" + echo "openjd_env: BLENDER_DAEMON_ADDRESS_2=$BLENDER_DAEMON_ADDRESS_2" +``` + +#### Run commands in the daemon + +Just like for starting a daemon, we will provide two options to run commands in the daemon. Each +command is equivalent to `blender --background ...`, but runs within the background daemon instead +of in a new Blender process. + +```sh +# Run commands in the daemon at address $BLENDER_DAEMON_ADDRESS +$ blender-openjd --openjd-daemon-run ... + +# Run commands in the specified address +$ blender-openjd --daemon-run ... +``` + +Here is example usage within a job template +[onRun](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#351-stepactions) +action. + +```yaml +... + actions: + onRun: + # This form lets you directly run commands in the daemon without manipulating any variables + command: blender-openjd + args: ["--openjd-daemon-run", "--render-frame", "{{Task.Param.Frame}}"] +``` + +```yaml +... + actions: + onRun: + command: "bash" + args: ["{{Env.File.OnRun}}"] + embeddedFiles: + - name: OnRun + filename: on-run.sh + type: TEXT + data: | + set -euo pipefail + + # This matches the default behavior of the --openjd-daemon-run option + blender-openjd --daemon-run "$BLENDER_DAEMON_ADDRESS" --render-frame "{{Task.Param.Frame}}" + + # You can use this form to run commands in multiple independent Blender background daemons. + blender-openjd --daemon-run "$BLENDER_DAEMON_ADDRESS_2" --render-frame "{{Task.Param.Frame}}" +``` + +#### Stop the daemon + +We provide two options to stop the daemon. One that uses the default environment variable name, and +a second that accepts the daemon address as an option. + +```sh +# Stop the daemon at address $BLENDER_DAEMON_ADDRESS +$ blender-openjd --openjd-daemon-stop + +# Stop the daemon at the specified address +$ blender-openjd --daemon-stop +``` + +Usage within a job template is straightforward. + +```yaml +... + actions: + onEnter: + command: blender-openjd + args: ["--openjd-daemon-stop"] +``` + +```yaml +... + actions: + OnExit: + command: "bash" + args: ["{{Env.File.OnExit}}"] + embeddedFiles: + - name: OnExit + filename: on-exit.sh + type: TEXT + data: | + set -euo pipefail + + # This matches the default behavior of the --openjd-daemon-stop option + blender-openjd --daemon-stop "$BLENDER_DAEMON_ADDRESS" + + # You can use this form to stop other Blender daemons + blender-openjd --daemon-stop "$BLENDER_DAEMON_ADDRESS_2" +``` + +#### Open Job Description progress and status messages + +We will provide one option to enable progress and status messages, `--openjd-updates`. Typically this will +be used immediately after the `--background` or `--openjd-daemon-start` options, to provide Open Job Description +progress updates for the lifetime of the Python process being launched, whether it runs as a single command or +as a daemon. + +When this option is enabled, rendering output will include `openjd_status` and `openjd_progress` messages. This +will not affect custom scripts, it is up to authors of those custom batch processing scripts to output +suitable status and progress messages. + +An example of what that might look like: + +```sh +$ blender-openjd --background '/.../blender-3.5-splash.blend' \ + --openjd-updates \ + --render-output '/.../output_####' \ + --render-format JPEG \ + --use-extension 1 \ + --render-frame 1 +Blender 4.4.0 (hash 05377985c527 built 2025-03-18 03:01:40) +openjd_status: Read blend: "/.../blender-3.5-splash.blend" +openjd_status: Preparing frame 1... +Fra:1 Mem:421.62M (Peak 422.43M) | Time:00:12.91 | Mem:0.00M, Peak:0.00M | Main, ViewLayer | Synchronizing object | Icosphere +... +Fra:1 Mem:564.29M (Peak 663.39M) | Time:00:15.28 | Mem:337.70M, Peak:340.61M | Main, ViewLayer | Loading denoising kernels (may take a few minutes the first time) +Fra:1 Mem:564.29M (Peak 663.39M) | Time:00:15.28 | Mem:337.70M, Peak:340.61M | Main, ViewLayer | Sample 0/128 +openjd_status: Rendering frame 1... +Fra:1 Mem:684.53M (Peak 684.53M) | Time:00:23.55 | Remaining:17:31.44 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Sample 1/128 +openjd_progress: 1 +... +Fra:1 Mem:684.53M (Peak 684.53M) | Time:07:41.36 | Remaining:10:35.98 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Sample 49/128 +openjd_progress: 38 +... +Fra:1 Mem:684.53M (Peak 684.53M) | Time:13:13.38 | Remaining:04:02.92 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Sample 96/128 +openjd_progress: 75 +... +Fra:1 Mem:735.16M (Peak 811.09M) | Time:17:27.41 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Sample 128/128 +openjd_progress: 100 +Fra:1 Mem:735.16M (Peak 811.09M) | Time:17:27.41 | Mem:457.94M, Peak:457.94M | Main, ViewLayer | Finished +Fra:1 Mem:292.20M (Peak 811.09M) | Time:17:27.44 | Compositing +openjd_status: Saved: '/.../output_0001.jpg' +Time: 17:27.67 (Saving: 00:00.17) +``` + +### Map file system paths + +We will provide an option `--openjd-path-mapping-rules ` to load path mapping metadata in +[the format specified by Open Job Description](https://github.com/OpenJobDescription/openjd-specifications/wiki/How-Jobs-Are-Run#path-mapping). +This metadata will apply to any later scene file loading operations, so it should be specified earlier than any `.blend` files provided +as part of the command. Typically this will be used immediately after the `--background` or `--openjd-daemon-start` options, or right +before the first `.blend` file specified at the command line. + +An example command in a job template may look like the following, with a template value substitution for the path mapping +rules that the job runtime gives it. + +```sh +# PATH_MAPPING_RULES comes from {{Session.PathMappingRulesFile}} in a job template +$ blender-openjd --background \ + --openjd-path-mapping-rules "$PATH_MAPPING_RULES" \ + '/.../blender-3.5-splash.blend' \ + --render-output '/.../output_####' \ + --render-format JPEG \ + --use-extension 1 \ + --render-frame 1 +``` + +### Render task chunks + +We will provide an option `--openjd-render-frame` that is identical to `--render-frame` but accepts frame numbers in the +[Open Job Description range expression](https://github.com/OpenJobDescription/openjd-specifications/wiki/2023-09-Template-Schemas#34111-intrangeexpr) +format instead of the usual Blender format. + +```sh +$ blender-openjd --background '/.../blender-3.5-splash.blend' \ + --render-output '/.../output_####' \ + --render-format JPEG \ + --use-extension 1 \ + --openjd-render-frame "1-5:2,9-11,15" +``` + +The equivalent command with `--render-frame` would be + +```sh +$ blender --background '/.../blender-3.5-splash.blend' \ + --render-output '/.../output_####' \ + --render-format JPEG \ + --use-extension 1 \ + --render-frame "1,3,5,9..11,15" +``` + +## Implementation and release strategy + +There is already a Blender adaptor with an interface different from this proposal. This adaptor is also the pilot +project to show a better way for all the adaptors to work, where we migrate the interfaces of each +`deadline-cloud-for-` project to follow a pattern extending the DCC's CLI interface replacing the +YAML-based options. + +Here's the current interface for `blender-openjd`: + +```sh +$ blender-openjd -h +usage: BlenderAdaptor [arguments] + +options: + -h, --help show this help message and exit + +commands: + {show-config,version-info,is-compatible,run,daemon} + show-config Prints the adaptor runtime configuration, then the program exits. + version-info Prints CLI and data interface versions, then the program exits. + is-compatible Validates compatiblity for the adaptor CLI interface and integration data interface provided + run Run through the start, run, stop, cleanup adaptor states. + daemon Runs the adaptor in a daemon mode. +``` + +Observe that none of the sub-commands overlap with the new proposed API, so we can make the command +provide both interfaces simultaneously. Our release strategy can therefore be a two-phase deployment: + +1. First we implement and release the new interface into the existing CLI command. Previous usage + of the adaptor remains unchanged, because it does not overlap with the new options. +2. Next we modify the submitter GUI that's integrated with Blender to support the new interface, + and release it behind a feature flag. +3. After thorough testing and validation, we change the submitter GUI default to use the new interface. +4. Finally, we remove the code for the old interface. From ca7c47ac14c1e57a04bba9d1c8278a8496ce06d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Jun 2025 18:21:55 +0000 Subject: [PATCH 38/39] chore(deps): update python-semantic-release requirement Updates the requirements on [python-semantic-release](https://github.com/python-semantic-release/python-semantic-release) to permit the latest version. - [Release notes](https://github.com/python-semantic-release/python-semantic-release/releases) - [Changelog](https://github.com/python-semantic-release/python-semantic-release/blob/master/CHANGELOG.rst) - [Commits](https://github.com/python-semantic-release/python-semantic-release/compare/v9.21...v10.0.2) --- updated-dependencies: - dependency-name: python-semantic-release dependency-version: 10.0.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Justin Blagden --- requirements-release.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-release.txt b/requirements-release.txt index 417ade73..8abfc12e 100644 --- a/requirements-release.txt +++ b/requirements-release.txt @@ -1 +1 @@ -python-semantic-release == 9.21.* \ No newline at end of file +python-semantic-release == 10.1.* \ No newline at end of file From 9ab0997f68c3de539c26cf56e3de538910aa1466 Mon Sep 17 00:00:00 2001 From: Justin Blagden Date: Thu, 21 Aug 2025 13:52:13 -0500 Subject: [PATCH 39/39] Signing unsigned commits Signed-off-by: Justin Blagden --- .../adaptor_runtime/unit/process/test_logging_subprocess.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py b/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py index cc57ad21..87ea6c07 100644 --- a/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py +++ b/test/openjd/adaptor_runtime/unit/process/test_logging_subprocess.py @@ -462,10 +462,7 @@ def test_command_printed(self, mock_popen: mock.Mock, caplog): args = ["cat", "foo.txt"] LoggingSubprocess(args=args) - if OSName.is_linux(): - assert "Running command: /usr/bin/cat foo.txt" in caplog.text - elif OSName.is_macos(): - assert "Running command: /bin/cat foo.txt" in caplog.text + assert "Running command: cat foo.txt" in caplog.text @mock.patch.object(logging_subprocess.subprocess, "Popen", autospec=True) def test_startup_directory_default(self, mock_popen_autospec: mock.Mock):