From 50437a98b9c9da060203de291345f78bbf4176e8 Mon Sep 17 00:00:00 2001 From: Brian Axelson <86568017+baxeaz@users.noreply.github.com> Date: Fri, 16 May 2025 21:35:15 +0000 Subject: [PATCH] feat: Adding support for redacted environment variable values through openjd_redacted_env Signed-off-by: Brian Axelson <86568017+baxeaz@users.noreply.github.com> --- src/openjd/model/v2023_09/_model.py | 38 ++++++----- .../openjd/model/v2023_09/test_definitions.py | 37 ++++++++++- test/openjd/model/v2023_09/test_module.py | 27 ++++++++ .../model/v2023_09/test_redacted_env_vars.py | 65 +++++++++++++++++++ 4 files changed, 148 insertions(+), 19 deletions(-) create mode 100644 test/openjd/model/v2023_09/test_module.py create mode 100644 test/openjd/model/v2023_09/test_redacted_env_vars.py diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index ab95fb11..e3dbea40 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -89,6 +89,8 @@ class ExtensionName(str, Enum): # # https://github.com/OpenJobDescription/openjd-specifications/blob/mainline/rfcs/0001-task-chunking.md TASK_CHUNKING = "TASK_CHUNKING" + # Extension that enables the use of openjd_redacted_env for setting environment variables with redacted values in logs + REDACTED_ENV_VARS = "REDACTED_ENV_VARS" ExtensionNameList = Annotated[list[str], Field(min_length=1)] @@ -512,24 +514,6 @@ class RangeString(FormatString): TaskRangeList = list[Union[TaskParameterStringValueAsJob, int, float, Decimal]] -# Target model for task parameters when instantiating a job. -class RangeListTaskParameterDefinition(OpenJDModel_v2023_09): - # element type of items in the range - type: TaskParameterType - # NOTE: Pydantic V1 was allowing non-string values in this range, V2 is enforcing that type. - range: TaskRangeList - # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension - chunks: Optional[TaskChunksDefinition] = None - - -class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): - # element type of items in the range - type: TaskParameterType - range: IntRangeExpr - # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension - chunks: Optional[TaskChunksDefinition] = None - - class TaskChunksRangeConstraint(str, Enum): CONTIGUOUS = "CONTIGUOUS" NONCONTIGUOUS = "NONCONTIGUOUS" @@ -559,6 +543,24 @@ def _validate_target_runtime_seconds(cls, value: Any, info: ValidationInfo) -> A return validate_int_fmtstring_field(value, ge=0, context=context) +# Target model for task parameters when instantiating a job. +class RangeListTaskParameterDefinition(OpenJDModel_v2023_09): + # element type of items in the range + type: TaskParameterType + # NOTE: Pydantic V1 was allowing non-string values in this range, V2 is enforcing that type. + range: TaskRangeList + # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension + chunks: Optional[TaskChunksDefinition] = None + + +class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): + # element type of items in the range + type: TaskParameterType + range: IntRangeExpr + # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension + chunks: Optional[TaskChunksDefinition] = None + + class IntTaskParameterDefinition(OpenJDModel_v2023_09): """Definition of an integer-typed Task Parameter and its value range. diff --git a/test/openjd/model/v2023_09/test_definitions.py b/test/openjd/model/v2023_09/test_definitions.py index 82f2a80c..8a9bb4cd 100644 --- a/test/openjd/model/v2023_09/test_definitions.py +++ b/test/openjd/model/v2023_09/test_definitions.py @@ -2,17 +2,37 @@ import pytest from pydantic import BaseModel -from typing import Type +from typing import Type, ForwardRef import openjd.model.v2023_09 as mod from inspect import getmembers, getmodule, isclass +from .test_module import ClassWithForwardRef, ClassWithoutForwardRef + + ALL_MODELS = sorted( [obj for name, obj in getmembers(mod) if isclass(obj) and issubclass(obj, BaseModel)], key=lambda o: o.__name__, ) +def test_forward_ref_detection(): + """Test that our ForwardRef detection works by checking classes that use ForwardRefs vs direct references.""" + # When referencing a class that's already defined, no ForwardRef is created + field = ClassWithoutForwardRef.model_fields["ref"] + assert not isinstance(field.annotation, ForwardRef), ( + "Expected ClassWithoutForwardRef.ref to NOT be a ForwardRef since ReferencedClass " + "is defined before it's used" + ) + + # When referencing a class that's defined later, a ForwardRef is created + field = ClassWithForwardRef.model_fields["ref"] + assert isinstance(field.annotation, ForwardRef), ( + "Expected ClassWithForwardRef.ref to be a ForwardRef since SecondReferencedClass " + "is defined after it's used" + ) + + @pytest.mark.parametrize("model", ALL_MODELS) def test_models_in_same_module(model: Type[BaseModel]) -> None: # For our error reporting of discriminated union fields to be correctly reported @@ -21,3 +41,18 @@ def test_models_in_same_module(model: Type[BaseModel]) -> None: # This is to identify when a name in an error location is actually a class name from # a typed union. assert getmodule(mod.JobTemplate) == getmodule(model) + + +@pytest.mark.parametrize("model", ALL_MODELS) +def test_no_forward_refs_in_models(model: Type[BaseModel]) -> None: + """Test that no models in _model.py use ForwardRefs in their field annotations. + + ForwardRefs indicate that a type is being referenced before it's defined, which can lead + to issues in pydantic validation. This test ensures all types are properly defined before + they're used. + """ + for field_name, field in model.model_fields.items(): + assert not isinstance(field.annotation, ForwardRef), ( + f"Field '{field_name}' in model '{model.__name__}' uses a ForwardRef. " + "The referenced type should be defined before it's used." + ) diff --git a/test/openjd/model/v2023_09/test_module.py b/test/openjd/model/v2023_09/test_module.py new file mode 100644 index 00000000..e91ef3fc --- /dev/null +++ b/test/openjd/model/v2023_09/test_module.py @@ -0,0 +1,27 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +from __future__ import annotations + +from typing import Optional +from openjd.model.v2023_09._model import OpenJDModel_v2023_09 + + +# First define a class we'll reference properly +class ReferencedClass(OpenJDModel_v2023_09): + value: str + + +class ClassWithoutForwardRef(OpenJDModel_v2023_09): + # This won't create a ForwardRef since ReferencedClass is already defined + ref: ReferencedClass + + +# Now try to reference a class before it's defined, like we did in _model.py +class ClassWithForwardRef(OpenJDModel_v2023_09): + # This should create a ForwardRef since SecondReferencedClass isn't defined yet + ref: Optional[SecondReferencedClass] = None + + +# Define the class after it's referenced +class SecondReferencedClass(OpenJDModel_v2023_09): + value: str diff --git a/test/openjd/model/v2023_09/test_redacted_env_vars.py b/test/openjd/model/v2023_09/test_redacted_env_vars.py new file mode 100644 index 00000000..8259da2e --- /dev/null +++ b/test/openjd/model/v2023_09/test_redacted_env_vars.py @@ -0,0 +1,65 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import pytest +from pydantic import ValidationError + +from openjd.model._parse import _parse_model +from openjd.model.v2023_09 import ( + JobTemplate, + ModelParsingContext, +) + + +def test_redacted_env_vars_extension_supported() -> None: + """Test that the REDACTED_ENV_VARS extension can be used in a job template.""" + data = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["REDACTED_ENV_VARS"], + "name": "Test Job", + "steps": [ + { + "name": "step1", + "script": { + "actions": {"onRun": {"command": "python", "args": ["{{Task.File.Run}}"]}}, + "embeddedFiles": [ + { + "name": "Run", + "type": "TEXT", + "data": 'print("openjd_redacted_env: SECRETVAR=SECRETVAL")', + } + ], + }, + } + ], + } + + # It parses successfully when the REDACTED_ENV_VARS extension is requested + _parse_model( + model=JobTemplate, + obj=data, + context=ModelParsingContext(supported_extensions=["REDACTED_ENV_VARS"]), + ) + + +def test_redacted_env_vars_extension_not_supported() -> None: + """Test that using REDACTED_ENV_VARS extension fails when not supported.""" + data = { + "specificationVersion": "jobtemplate-2023-09", + "extensions": ["REDACTED_ENV_VARS"], + "name": "Test Job", + "steps": [ + { + "name": "step1", + "script": {"actions": {"onRun": {"command": "echo", "args": ["test"]}}}, + } + ], + } + + # It fails to parse when REDACTED_ENV_VARS extension is not supported + with pytest.raises(ValidationError) as excinfo: + _parse_model( + model=JobTemplate, + obj=data, + context=ModelParsingContext(), + ) + assert "Unsupported extension names: REDACTED_ENV_VARS" in str(excinfo.value)