From f4f9db96428bb6f258282fd7dc29501883dc86a4 Mon Sep 17 00:00:00 2001 From: Mark Wiebe <399551+mwiebe@users.noreply.github.com> Date: Mon, 24 Mar 2025 15:19:55 -0700 Subject: [PATCH] refactor!: Update FormatString expression to accept parsing context To support Open Job Description modeling parsing that differs depending on the revision and/or enabled extensions, the code needs a way to parse differently based on those values. The chosen approach is to include these in the parsing context. * Introduce an abstract base class, ModelParsingContextInterface, that every model parsing context must be a subclass of. It has variables spec_rev and extensions, for parsing to reference. * Modify the DynamicConstrainedStr class to require a ModelParsingContextInterface during construction. The FormatString to handle template the expression syntax is a subset, so modifying this class to handle the Pydantic integration was the simplest approach. * As part of this, change DynamicConstrainedStr and subclasses to use __new__ instead of __init__, so they all handle construction uniformly with the required context parameter. * Everywhere that the tests do model validation or construct DynamicConstrainedStr/FormatString instances, introduce the model parsing context. In some places had to change the content of the tests to reflect the more strict separation between model parsing and job instantiation. * Change various different model construction used in the tests to call decode_job_template instead. The latter function has its own tests, and using the specified Open Job Description syntax instead of object constructors isolates the tests from implementation details. * Create a function format_string_expr_parse that handles parsing a string according to the format string expression parsing syntax, using a model parsing context to determine the specific syntax. This does not change the syntax, only introduces a place to add new syntax support. * Update the template variable reference pre-validation and template -> job instantiation code to work with the updated DynamicConstrainedStr that now requires a context. This required the instantiation logic to be more clear about the separation between Template and Job, since the latter instantiation happens without a context. * Simplify the instantiate_model logic with a context manager to unify handling of validation errors. This reduces code duplication, and removed the optional loc and within_field arguments that were for internal recursion purposes. BREAKING CHANGE: Any creation of a DynamicConstrainedStr or FormatString will need to provide a model parsing context, that includes the Open Job Description revision and any extensions that are enabled. Use of the instantiate_model can no longer provide optional loc and within_field arguments. Signed-off-by: Mark Wiebe <399551+mwiebe@users.noreply.github.com> --- .../_format_strings/_dyn_constrained_str.py | 25 +- .../model/_format_strings/_expression.py | 8 +- .../model/_format_strings/_format_string.py | 21 +- src/openjd/model/_format_strings/_parser.py | 21 +- src/openjd/model/_internal/_create_job.py | 324 ++- .../model/_internal/_validator_functions.py | 46 +- .../_variable_reference_validation.py | 55 +- src/openjd/model/_parse.py | 4 +- src/openjd/model/_step_param_space_iter.py | 11 +- src/openjd/model/_types.py | 42 +- src/openjd/model/v2023_09/_model.py | 161 +- .../openjd/model/_internal/test_create_job.py | 108 +- .../test_variable_reference_validation.py | 142 +- .../test_dyn_constrained_str.py | 55 +- .../model/format_strings/test_expression.py | 21 +- .../format_strings/test_format_string.py | 15 +- .../model/format_strings/test_parser.py | 26 +- test/openjd/model/test_capabilities.py | 22 +- test/openjd/model/test_create_job.py | 316 +-- test/openjd/model/v2023_09/test_create.py | 563 ++--- test/openjd/model/v2023_09/test_strings.py | 71 +- .../model/v2023_09/test_template_variables.py | 1882 ++++++++--------- 22 files changed, 2142 insertions(+), 1797 deletions(-) diff --git a/src/openjd/model/_format_strings/_dyn_constrained_str.py b/src/openjd/model/_format_strings/_dyn_constrained_str.py index 20c32a52..fa43c3b9 100644 --- a/src/openjd/model/_format_strings/_dyn_constrained_str.py +++ b/src/openjd/model/_format_strings/_dyn_constrained_str.py @@ -2,16 +2,23 @@ from typing import Any, Callable, Optional, Pattern, Union -from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler +from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, ValidationInfo from pydantic.json_schema import JsonSchemaValue from pydantic_core import core_schema import re +from .._types import ModelParsingContextInterface + class DynamicConstrainedStr(str): """Constrained string type for interfacing with Pydantic. The maximum string length can be dynamically defined at runtime. + The parsing context, a subclass of ModelParsingContextInterface, + is required to construct a DynamicConstrainedStr or subclass. + This enables the FormatString to handle the supported expression types + based on the Open Job Description revision version and extensions. + Note: Does *not* run model validation when constructed. """ @@ -23,6 +30,9 @@ class DynamicConstrainedStr(str): # ================================ # Reference: https://pydantic-docs.helpmanual.io/usage/types/#custom-data-types + def __new__(cls, value: str, *, context: ModelParsingContextInterface): + return super().__new__(cls, value) + @classmethod def _get_max_length(cls) -> Optional[int]: if callable(cls._max_length): @@ -30,7 +40,7 @@ def _get_max_length(cls) -> Optional[int]: return cls._max_length @classmethod - def _validate(cls, value: str) -> Any: + def _validate(cls, value: str, info: ValidationInfo) -> Any: if not isinstance(value, str): raise ValueError("String required") @@ -46,13 +56,20 @@ def _validate(cls, value: str) -> Any: pattern: str = cls._regex if isinstance(cls._regex, str) else cls._regex.pattern raise ValueError(f"String does not match the required pattern: {pattern}") - return cls(value) + if type(value) is cls: + return value + else: + if info.context is None: + raise ValueError( + f"Internal parsing error: No parsing context was provided during model validation for the DynamicConstrainedStr subclass {cls.__name__}." + ) + return cls(value, context=info.context) @classmethod def __get_pydantic_core_schema__( cls, source_type: type[Any], handler: GetCoreSchemaHandler ) -> core_schema.CoreSchema: - return core_schema.no_info_plain_validator_function(cls._validate) + return core_schema.with_info_plain_validator_function(cls._validate) @classmethod def __get_pydantic_json_schema__( diff --git a/src/openjd/model/_format_strings/_expression.py b/src/openjd/model/_format_strings/_expression.py index 31a5887d..85d10a8b 100644 --- a/src/openjd/model/_format_strings/_expression.py +++ b/src/openjd/model/_format_strings/_expression.py @@ -6,14 +6,15 @@ from .._errors import ExpressionError from .._symbol_table import SymbolTable from ._nodes import Node -from ._parser import Parser +from ._parser import parse_format_string_expr +from .._types import ModelParsingContextInterface class InterpolationExpression: expr: str _expression_tree: Node - def __init__(self, expr: str) -> None: + def __init__(self, expr: str, *, context: ModelParsingContextInterface) -> None: """Constructor. Raises: @@ -24,10 +25,9 @@ def __init__(self, expr: str) -> None: expr (str): The expression """ self.expr = expr - parser = Parser() # Raises: ExpressionError, TokenError - self._expresion_tree = parser.parse(expr) + self._expresion_tree = parse_format_string_expr(expr, context=context) def validate_symbol_refs(self, *, symbols: set[str]) -> None: """Check whether this expression can be evaluated correctly given a set of symbol names. diff --git a/src/openjd/model/_format_strings/_format_string.py b/src/openjd/model/_format_strings/_format_string.py index 42f35b90..4307e7aa 100644 --- a/src/openjd/model/_format_strings/_format_string.py +++ b/src/openjd/model/_format_strings/_format_string.py @@ -8,6 +8,7 @@ from .._symbol_table import SymbolTable from ._dyn_constrained_str import DynamicConstrainedStr from ._expression import InterpolationExpression +from .._types import ModelParsingContextInterface @dataclass @@ -30,12 +31,11 @@ def __init__(self, *, string: str, start: int, end: int, expr: str = "", details ) super().__init__(msg) - def __str__(self) -> str: - return self.args[0] - class FormatString(DynamicConstrainedStr): - def __init__(self, value: str): + _processed_list: list[Union[str, ExpressionInfo]] + + def __new__(cls, value: str, *, context: ModelParsingContextInterface): """ Instantiate a FormatString from a given string. @@ -55,8 +55,9 @@ def __init__(self, value: str): ------ FormatStringError: if the original string is nonvalid. """ - # Note: str is constructed in __new__, so don't call super __init__ - self._processed_list: list[Union[str, ExpressionInfo]] = self._preprocess() + self = super().__new__(cls, value, context=context) + self._processed_list = self._preprocess(context=context) + return self @property def original_value(self) -> str: @@ -125,7 +126,9 @@ def resolve(self, *, symtab: SymbolTable) -> str: return "".join(resolved_list) - def _preprocess(self) -> list[Union[str, ExpressionInfo]]: + def _preprocess( + self, *, context: ModelParsingContextInterface + ) -> list[Union[str, ExpressionInfo]]: """ Scans through the original string to find all interpolation expressions inside of {{ }}. Also, validates the content of each interpolation expression inside of {{ }}. @@ -187,7 +190,9 @@ def _preprocess(self) -> list[Union[str, ExpressionInfo]]: expression_info = ExpressionInfo(braces_start, braces_end) try: - expr = InterpolationExpression(self[expression_start:expression_end]) + expr = InterpolationExpression( + self[expression_start:expression_end], context=context + ) except (ExpressionError, TokenError) as exc: raise FormatStringError( string=self.original_value, diff --git a/src/openjd/model/_format_strings/_parser.py b/src/openjd/model/_format_strings/_parser.py index 70d9b9b7..1a49a36f 100644 --- a/src/openjd/model/_format_strings/_parser.py +++ b/src/openjd/model/_format_strings/_parser.py @@ -6,13 +6,30 @@ from .._tokenstream import Token, TokenStream, TokenType from ._nodes import FullNameNode, Node from ._tokens import DotToken, NameToken +from .._types import ModelParsingContextInterface _tokens: dict[TokenType, Type[Token]] = {TokenType.NAME: NameToken, TokenType.DOT: DotToken} -class Parser: +def parse_format_string_expr(expr: str, *, context: ModelParsingContextInterface) -> Node: + """Generate an expression tree for the given string interpolation expression. + + Args: + expr (str): A string interpolation expression + + Raises: + ExpressionError: If the given expression does not adhere to the grammar. + TokenError: If the given expression contains nonvalid or unexpected tokens. + + Returns: + Node: Root of the expression tree. + """ + return FormatStringExprParser_v2023_09().parse(expr) + + +class FormatStringExprParser_v2023_09: """ - Parser used to build an AST of the currently supported operations. + Parser used to build an AST of format strings for the 2023-09 specification. """ def parse(self, expr: str) -> Node: diff --git a/src/openjd/model/_internal/_create_job.py b/src/openjd/model/_internal/_create_job.py index 3f1f5acc..2b5094e5 100644 --- a/src/openjd/model/_internal/_create_job.py +++ b/src/openjd/model/_internal/_create_job.py @@ -1,22 +1,56 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -from typing import cast, Any, Union +from contextlib import contextmanager +from typing import Annotated, Any, Union from pydantic import ValidationError +from pydantic import TypeAdapter from pydantic_core import InitErrorDetails from .._symbol_table import SymbolTable -from .._format_strings import FormatString, FormatStringError +from .._format_strings import FormatString from .._types import OpenJDModel __all__ = ("instantiate_model",) +@contextmanager +def capture_validation_errors( + *, output_errors: list[InitErrorDetails], loc: tuple[Union[str, int], ...], input: Any +): + """Context manager to collect validation errors from pydantic into a list. + + Args: + output_error_list (list[InitErrorDetails]): The list to collect errors into. + loc (tuple): The location of the input value being validated. + input (Any): The input value being validated. + """ + try: + yield + except ValidationError as exc: + # Convert the ErrorDetails to InitErrorDetails by extending the 'loc' and excluding the 'msg' + for error_details in exc.errors(): + init_error_details: dict[str, Any] = {} + for err_key, err_value in error_details.items(): + if err_key == "loc": + init_error_details["loc"] = (*loc, *err_value) # type: ignore + elif err_key != "msg": + init_error_details[err_key] = err_value + output_errors.append(init_error_details) # type: ignore + except ValueError as exc: + output_errors.append( + InitErrorDetails( + type="value_error", + loc=loc, + ctx={"error": exc}, + input=input, + ) + ) + + def instantiate_model( # noqa: C901 model: OpenJDModel, symtab: SymbolTable, - loc: tuple[Union[str, int], ...] = tuple[str](), - within_field: str = "", ) -> OpenJDModel: """This function is for instantiating a Template model into a Job model. @@ -27,9 +61,6 @@ def instantiate_model( # noqa: C901 model (OpenJDModel): The model instance to transform. symtab (SymbolTable): The symbol table containing fully qualified Job parameter values used during the instantiation. - loc (tuple[Union[str,int], ...], optional): Path to the model. Used for generating contextual errors. - within_field (str): The name of the field where this model was found during traversal. "" if this - is the top-level/root model Raises: ValidationError - If there are any validation errors from the target models. @@ -40,257 +71,182 @@ def instantiate_model( # noqa: C901 errors = list[InitErrorDetails]() instantiated_fields = dict[str, Any]() + # Determine the target model to create as + target_model = model.__class__ + if model._job_creation_metadata.create_as is not None: + create_as_metadata = model._job_creation_metadata.create_as + if create_as_metadata.model is not None: + target_model = create_as_metadata.model + elif create_as_metadata.callable is not None: + target_model = create_as_metadata.callable(model) + for field_name in model.model_fields.keys(): - target_field_name = field_name - if field_name in model._job_creation_metadata.rename_fields: - target_field_name = model._job_creation_metadata.rename_fields[field_name] if field_name in model._job_creation_metadata.exclude_fields: # The field is marked for being excluded continue + target_field_name = model._job_creation_metadata.rename_fields.get(field_name, field_name) + target_field_type: Any = target_model.model_fields[target_field_name].annotation + for metadata in target_model.model_fields[target_field_name].metadata: + target_field_type = Annotated[target_field_type, metadata] + if not hasattr(model, field_name): # Field has no value. Set to None and move on. instantiated_fields[target_field_name] = None continue - field = getattr(model, field_name) - instantiated: Any - # TODO - try/except. Collect errors - try: - if isinstance(field, list): - # Raises: ValidationError - instantiated = _instantiate_list_field(model, field_name, field, symtab, loc) - elif isinstance(field, dict): - # Raises: ValidationError - instantiated = _instantiate_dict_field(model, field_name, field, symtab, loc) + field_value = getattr(model, field_name) + instantiated: Any = None + with capture_validation_errors(output_errors=errors, loc=(field_name,), input=field_value): + # Instantiate and resolve format string expressions + needs_resolve = field_name in model._job_creation_metadata.resolve_fields + if isinstance(field_value, list): + if field_name in model._job_creation_metadata.reshape_field_to_dict: + key_field = model._job_creation_metadata.reshape_field_to_dict[field_name] + instantiated = _instantiate_list_field_as_dict( + field_value, symtab, needs_resolve, key_field + ) + else: + instantiated = _instantiate_list_field_as_list( + field_value, symtab, needs_resolve + ) + elif isinstance(field_value, dict): + instantiated = _instantiate_dict_field(field_value, symtab, needs_resolve) else: - # Raises: ValidationError, FormatStringError - instantiated = _instantiate_noncollection_value( - model, field_name, field, symtab, (*loc, field_name) - ) + instantiated = _instantiate_noncollection_value(field_value, symtab, needs_resolve) + + # Validate as the target field type + type_adaptor: Any = TypeAdapter(target_field_type) + instantiated = type_adaptor.validate_python(instantiated) instantiated_fields[target_field_name] = instantiated - except ValidationError as exc: - # Convert the ErrorDetails to InitErrorDetails by excluding the 'msg' - for error_details in exc.errors(): - init_error_details = { - key: value for key, value in error_details.items() if key != "msg" - } - errors.append(cast(InitErrorDetails, init_error_details)) - except FormatStringError as exc: - errors.append( - InitErrorDetails( - type="value_error", - loc=loc, - ctx={"error": ValueError(str(exc))}, - input=exc.input, - ) - ) - if errors: - raise ValidationError.from_exception_data( - title=model.__class__.__name__, line_errors=errors - ) + if not errors: + if model._job_creation_metadata.adds_fields is not None: + new_fields = model._job_creation_metadata.adds_fields(model, symtab) + instantiated_fields.update(**new_fields) - if model._job_creation_metadata.adds_fields is not None: - new_fields = model._job_creation_metadata.adds_fields(within_field, model, symtab) - instantiated_fields.update(**new_fields) + with capture_validation_errors(output_errors=errors, loc=(), input=field_value): + result = target_model(**instantiated_fields) - try: - if model._job_creation_metadata.create_as is not None: - create_as_metadata = model._job_creation_metadata.create_as - if create_as_metadata.model is not None: - return create_as_metadata.model(**instantiated_fields) - elif create_as_metadata.callable is not None: - create_as_class = create_as_metadata.callable(model) - return create_as_class(**instantiated_fields) - return model.__class__(**instantiated_fields) - except ValidationError as exc: - # Convert the ErrorDetails to InitErrorDetails by concatenating the 'loc' values and excluding the 'msg' - for error_details in exc.errors(): - init_error_details = {} - for key, value in error_details.items(): - if key == "loc": - init_error_details["loc"] = loc + cast(tuple, value) - elif key != "msg": - init_error_details[key] = value - errors.append(cast(InitErrorDetails, init_error_details)) + if errors: raise ValidationError.from_exception_data( title=model.__class__.__name__, line_errors=errors ) + return result + def _instantiate_noncollection_value( - within_model: OpenJDModel, - field_name: str, value: Any, symtab: SymbolTable, - loc: tuple[Union[str, int], ...], + needs_resolve: bool, ) -> Any: """Instantiate a single value that must not be a collection type (list, dict, etc). Arguments: within_model (OpenJDModel): The model within which the value is located. - field_name (str): The name of the field within that model that contains the value. value (Any): Value to process. symtab (SymbolTable): Symbol table for format string value lookups. - loc (tuple[Union[str,int], ...]): Path to this value. + needs_resolve (bool): Whether to resolve the value as a format string. """ - - # Note: Let the exceptions fall through to the calling context to handle. - # If we wrap them into ValidationErrors here, then the locations of errors will - # be incorrect. - if isinstance(value, OpenJDModel): - # Raises: ValidationError - return instantiate_model(value, symtab, loc, field_name) - elif ( - isinstance(value, FormatString) - and field_name in within_model._job_creation_metadata.resolve_fields - ): - # Raises: FormatStringError - return value.resolve(symtab=symtab) + return instantiate_model(value, symtab) + elif isinstance(value, FormatString) and needs_resolve: + value = value.resolve(symtab=symtab) return value -def _instantiate_list_field( # noqa: C901 - within_model: OpenJDModel, - field_name: str, +def _instantiate_list_field_as_list( # noqa: C901 value: list[Any], symtab: SymbolTable, - loc: tuple[Union[str, int], ...], -) -> Union[list[Any], dict[str, Any]]: + needs_resolve: bool, +) -> list[Any]: """As _instantiate_noncollection_value, but where the value is a list. Arguments: within_model (OpenJDModel): The model within which the value is located. - field_name (str): The name of the field within that model that contains the value. value (Any): Value to process. symtab (SymbolTable): Symbol table for format string value lookups. - loc (tuple[Union[str,int], ...]): Path to this value. + needs_resolve (bool): Whether to resolve the value as a format string. """ - errors = list[InitErrorDetails]() - result: Union[list[Any], dict[str, Any]] - if field_name in within_model._job_creation_metadata.reshape_field_to_dict: - key_field = within_model._job_creation_metadata.reshape_field_to_dict[field_name] - result = dict[str, Any]() - for idx, item in enumerate(value): - key = getattr(item, key_field) - try: - # Raises: ValidationError, FormatStringError - result[key] = _instantiate_noncollection_value( - within_model, - field_name, + errors: list[InitErrorDetails] = [] + result: list[Any] = [] + for idx, item in enumerate(value): + with capture_validation_errors(output_errors=errors, loc=(idx,), input=value): + # Raises: ValidationError, FormatStringError + result.append( + _instantiate_noncollection_value( item, symtab, - (*loc, field_name, idx), - ) - except ValidationError as exc: - # Convert the ErrorDetails to InitErrorDetails by excluding the 'msg' - for error_details in exc.errors(): - init_error_details = { - key: value for key, value in error_details.items() if key != "msg" - } - errors.append(cast(InitErrorDetails, init_error_details)) - except FormatStringError as exc: - errors.append( - InitErrorDetails( - type="value_error", - loc=loc, - ctx={"error": ValueError(str(exc))}, - input=exc.input, - ) - ) - else: - result = list[Any]() - for idx, item in enumerate(value): - try: - # Raises: ValidationError, FormatStringError - result.append( - _instantiate_noncollection_value( - within_model, - field_name, - item, - symtab, - (*loc, field_name, idx), - ) - ) - except ValidationError as exc: - # Convert the ErrorDetails to InitErrorDetails by excluding the 'msg' - for error_details in exc.errors(): - init_error_details = { - key: value for key, value in error_details.items() if key != "msg" - } - errors.append(cast(InitErrorDetails, init_error_details)) - except FormatStringError as exc: - errors.append( - InitErrorDetails( - type="value_error", - loc=loc, - ctx={"error": ValueError(str(exc))}, - input=exc.input, - ) + needs_resolve, ) + ) if errors: raise ValidationError.from_exception_data( - title=within_model.__class__.__name__, line_errors=errors + title="_instantiate_list_field_as_list", line_errors=errors + ) + + return result + + +def _instantiate_list_field_as_dict( # noqa: C901 + value: list[Any], symtab: SymbolTable, needs_resolve: bool, key_field: str +) -> dict[str, Any]: + """As _instantiate_noncollection_value, but where the value is a list. + + Arguments: + within_model (OpenJDModel): The model within which the value is located. + value (Any): Value to process. + symtab (SymbolTable): Symbol table for format string value lookups. + needs_resolve (bool): Whether to resolve the value as a format string. + key_field (str): The name of the key in each object that defines the output dictionary key. + """ + errors: list[InitErrorDetails] = [] + result: dict[str, Any] = {} + for idx, item in enumerate(value): + key = getattr(item, key_field) + with capture_validation_errors(output_errors=errors, loc=(idx,), input=value): + result[key] = _instantiate_noncollection_value( + item, + symtab, + needs_resolve, + ) + + if errors: + raise ValidationError.from_exception_data( + title="_instantiate_list_field_as_dict", line_errors=errors ) return result def _instantiate_dict_field( - within_model: OpenJDModel, - field_name: str, value: dict[str, Any], symtab: SymbolTable, - loc: tuple[Union[str, int], ...], + needs_resolve: bool, ) -> dict[str, Any]: """As _instantiate_noncollection_value, but where the value is a dict. Arguments: within_model (OpenJDModel): The model within which the value is located. - field_name (str): The name of the field within that model that contains the value. value (Any): Value to process. symtab (SymbolTable): Symbol table for format string value lookups. - loc (tuple[Union[str,int], ...]): Path to this value. + needs_resolve (bool): Whether to resolve the value as a format string. """ - errors = list[InitErrorDetails]() - result = dict[str, Any]() + errors: list[InitErrorDetails] = [] + result: dict[str, Any] = {} for key, item in value.items(): - try: - # Raises: ValidationError, FormatStringError + with capture_validation_errors(output_errors=errors, loc=(key,), input=value): result[key] = _instantiate_noncollection_value( - within_model, - key, # We call the dictionary key the field name for adds_fields arguments to be correct item, symtab, - loc - + ( - field_name, - key, - ), - ) - except ValidationError as exc: - # Convert the ErrorDetails to InitErrorDetails by excluding the 'msg' - for error_details in exc.errors(): - init_error_details = { - key: value for key, value in error_details.items() if key != "msg" - } - errors.append(cast(InitErrorDetails, init_error_details)) - except FormatStringError as exc: - errors.append( - InitErrorDetails( - type="value_error", - loc=loc, - ctx={"error": ValueError(str(exc))}, - input=exc.input, - ) + needs_resolve, ) if errors: raise ValidationError.from_exception_data( - title=within_model.__class__.__name__, line_errors=errors + title="_instantiate_dict_field", line_errors=errors ) return result diff --git a/src/openjd/model/_internal/_validator_functions.py b/src/openjd/model/_internal/_validator_functions.py index b7abd446..90065d06 100644 --- a/src/openjd/model/_internal/_validator_functions.py +++ b/src/openjd/model/_internal/_validator_functions.py @@ -6,18 +6,28 @@ from pydantic_core import PydanticKnownError, ValidationError, InitErrorDetails from .._format_strings import FormatString +from .._types import ModelParsingContextInterface def validate_int_fmtstring_field( - value: Union[int, float, Decimal, str, FormatString], ge: Optional[int] = None -) -> Union[int, float, Decimal, str, FormatString]: + value: Union[int, float, Decimal, str, FormatString], + ge: Optional[int] = None, + *, + context: Optional[ModelParsingContextInterface], +) -> Union[int, float, Decimal, FormatString]: """Validates a field that is allowed to be either an integer, a string containing an integer, or a string containing expressions that resolve to an integer.""" value_type_wrong_msg = "Value must be an integer or a string containing an integer." + if type(value) is str: + # If processing is being done without a context, the value must already be a FormatString + if context is None: + raise ValueError( + "Internal parsing error: No parsing context was provided during model validation for the FormatString." + ) + value = FormatString(value, context=context) + # Validate the type - if isinstance(value, str): - if not isinstance(value, FormatString): - value = FormatString(value) + if isinstance(value, FormatString): # If the string value has no expressions, we can validate the value now. if len(value.expressions) == 0: try: @@ -43,15 +53,25 @@ def validate_int_fmtstring_field( def validate_float_fmtstring_field( - value: Union[int, float, Decimal, str, FormatString], ge: Optional[Decimal] = None -) -> Union[int, float, Decimal, str, FormatString]: + value: Union[int, float, Decimal, str, FormatString], + ge: Optional[Decimal] = None, + *, + context: Optional[ModelParsingContextInterface], +) -> Union[int, float, Decimal, FormatString]: """Validates a field that is allowed to be either an float, a string containing an float, or a string containing expressions that resolve to a float.""" value_type_wrong_msg = "Value must be a float or a string containing a float." + + if type(value) is str: + # If processing is being done without a context, the value must already be a FormatString + if context is None: + raise ValueError( + "Internal parsing error: No parsing context was provided during model validation for the FormatString" + ) + value = FormatString(value, context=context) + # Validate the type - if isinstance(value, str): - if not isinstance(value, FormatString): - value = FormatString(value) + if isinstance(value, FormatString): # If the string value has no expressions, we can validate the value now. if len(value.expressions) == 0: try: @@ -77,12 +97,14 @@ def validate_float_fmtstring_field( return float_value -def validate_list_field(value: list, validator: Callable) -> list: +def validate_list_field( + value: list, validator: Callable, *, context: Optional[ModelParsingContextInterface] +) -> list: """Validates a list of values using the provided validator function.""" errors = list[InitErrorDetails]() for i, item in enumerate(value): try: - validator(item) + validator(item, context=context) except PydanticKnownError as exc: # Copy known errors verbatim with added location data errors.append( diff --git a/src/openjd/model/_internal/_variable_reference_validation.py b/src/openjd/model/_internal/_variable_reference_validation.py index 7b23a81e..2113bfdf 100644 --- a/src/openjd/model/_internal/_variable_reference_validation.py +++ b/src/openjd/model/_internal/_variable_reference_validation.py @@ -12,7 +12,7 @@ # Workaround for Python 3.9 where issubclass raises an error "TypeError: issubclass() arg 1 must be a class" from pydantic.v1.utils import lenient_issubclass -from .._types import OpenJDModel, ResolutionScope +from .._types import OpenJDModel, ResolutionScope, ModelParsingContextInterface from .._format_strings import FormatString, FormatStringError __all__ = ["prevalidate_model_template_variable_references"] @@ -130,7 +130,12 @@ def update_self(self, other: "ScopedSymtabs") -> "ScopedSymtabs": return self -def prevalidate_model_template_variable_references(cls: Type[OpenJDModel], values: dict[str, Any]): +def prevalidate_model_template_variable_references( + cls: Type[OpenJDModel], + values: dict[str, Any], + *, + context: Optional[ModelParsingContextInterface], +): """Validates the template variable references in a given model. Notes: @@ -151,6 +156,7 @@ def prevalidate_model_template_variable_references(cls: Type[OpenJDModel], value symbols=ScopedSymtabs(), loc=(), symbol_prefix="", + context=context, ) @@ -185,6 +191,8 @@ def _validate_model_template_variable_references( symbols: ScopedSymtabs, loc: tuple, discriminator: Union[str, Discriminator, None] = None, + *, + context: Optional[ModelParsingContextInterface], ) -> list[InitErrorDetails]: """Inner implementation of prevalidate_model_template_variable_references(). @@ -212,6 +220,7 @@ def _validate_model_template_variable_references( symbols, loc, discriminator=discriminator, + context=context, ) # Unwrap the Annotated type, and get the discriminator while doing so @@ -228,6 +237,7 @@ def _validate_model_template_variable_references( symbols, loc, discriminator=discriminator, + context=context, ) # Validate all the items of a list @@ -239,7 +249,13 @@ def _validate_model_template_variable_references( for i, item in enumerate(value): errors.extend( _validate_model_template_variable_references( - item_model, item, current_scope, symbol_prefix, symbols, (*loc, i) + item_model, + item, + current_scope, + symbol_prefix, + symbols, + (*loc, i), + context=context, ) ) return errors @@ -260,6 +276,7 @@ def _validate_model_template_variable_references( symbol_prefix, symbols, (*loc, key), + context=context, ) ) return errors @@ -269,7 +286,7 @@ def _validate_model_template_variable_references( for sub_type in typing.get_args(model): errors.extend( _validate_model_template_variable_references( - sub_type, value, current_scope, symbol_prefix, symbols, loc + sub_type, value, current_scope, symbol_prefix, symbols, loc, context=context ) ) return errors @@ -285,14 +302,23 @@ def _validate_model_template_variable_references( symbol_prefix, symbols, loc, + context=context, ) else: - return [] + return errors if isclass(model) and lenient_issubclass(model, FormatString): - if isinstance(value, str): - return _check_format_string(value, current_scope, symbols, loc) - return [] + if not isinstance(value, FormatString): + if context is None: + raise ValueError( + "Internal parsing error: No parsing context was provided during model validation for the FormatString" + ) + try: + value = FormatString(value, context=context) + except FormatStringError: + # Do not add an error for this during prevalidation, this will be caught later. + return errors + return _check_format_string(value, current_scope, symbols, loc, context=context) # Return an empty error list if it's not an OpenJDModel, or if it's not a dict if not (isclass(model) and lenient_issubclass(model, OpenJDModel) and isinstance(value, dict)): @@ -349,6 +375,7 @@ def _validate_model_template_variable_references( validation_symbols, (*loc, field_name), field_info.discriminator, + context=context, ) ) @@ -356,14 +383,21 @@ def _validate_model_template_variable_references( def _check_format_string( - value: str, current_scope: ResolutionScope, symbols: ScopedSymtabs, loc: tuple + value: FormatString, + current_scope: ResolutionScope, + symbols: ScopedSymtabs, + loc: tuple, + context: Optional[ModelParsingContextInterface], ) -> list[InitErrorDetails]: # Collect the variable reference errors, if any, from the given FormatString value. errors = list[InitErrorDetails]() scoped_symbols = symbols[current_scope] try: - f_value = FormatString(value) + if isinstance(value, FormatString): + f_value = value + else: + f_value = FormatString(value, context=context) except FormatStringError: # Improperly formed string. Later validation passes will catch and flag this. return errors @@ -457,7 +491,6 @@ def _collect_variable_definitions( # noqa: C901 (suppress: too complex) When the model is not an OpenJDModel, it only populates the "__export__". """ - # NOTE: This is not written to be super generic and handle all possible OpenJD models going # forward forever. It handles the subset of the general Pydantic data model that OpenJD is # currently using, and will be extended as we use additional features of Pydantic's data model. diff --git a/src/openjd/model/_parse.py b/src/openjd/model/_parse.py index 0b8f4dd6..a78b153a 100644 --- a/src/openjd/model/_parse.py +++ b/src/openjd/model/_parse.py @@ -57,11 +57,11 @@ def _parse_model(*, model: Type[T], obj: Any, context: Any = None) -> T: prevalidator_error: Optional[PydanticValidationError] = None if hasattr(model, "_root_template_prevalidator"): try: - getattr(model, "_root_template_prevalidator")(obj) + getattr(model, "_root_template_prevalidator")(obj, context=context) except PydanticValidationError as exc: prevalidator_error = exc try: - result = cast(T, cast(BaseModel, model).model_validate(obj, context=context)) + result = model.model_validate(obj, context=context) # type: ignore except PydanticValidationError as exc: if prevalidator_error is not None: errors = list[InitErrorDetails]() diff --git a/src/openjd/model/_step_param_space_iter.py b/src/openjd/model/_step_param_space_iter.py index 106ca1bb..d3aec120 100644 --- a/src/openjd/model/_step_param_space_iter.py +++ b/src/openjd/model/_step_param_space_iter.py @@ -5,6 +5,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Sized from dataclasses import dataclass, field +from decimal import Decimal from functools import reduce from operator import mul import re @@ -328,7 +329,7 @@ def _create_expr_tree( return RangeListIdentifierNode( name=name, type=ParameterValueType(parameter.type), - range=chunk_list, + range=chunk_list, # type: ignore range_set=set(parameter_range), range_constraint=parameter.chunks.rangeConstraint, ) @@ -772,7 +773,7 @@ class RangeListIdentifierNodeIterator(NodeIterator): _node: The RangeListIdentifierNode this is iterating over. """ - _it: Iterator[str] + _it: Iterator[Union[str, int, float, Decimal]] _node: RangeListIdentifierNode def __init__(self, node: RangeListIdentifierNode): @@ -785,7 +786,7 @@ def reset_iter(self) -> None: def next(self, result: TaskParameterSet) -> None: # Raises: StopIteration v = next(self._it) - result[self._node.name] = ParameterValue(type=self._node.type, value=v) + result[self._node.name] = ParameterValue(type=self._node.type, value=str(v)) INTERVAL_RE = re.compile(r"\s*(-?[0-9]+)\s*-\s*(-?[0-9]+)\s*") @@ -795,7 +796,7 @@ def next(self, result: TaskParameterSet) -> None: class RangeListIdentifierNode(Node): name: str type: ParameterValueType - range: list[str] + range: list[Union[str, int, float, Decimal]] range_set: set range_constraint: Optional[TaskChunksRangeConstraint_2023_09] = None _len: int = field(init=False, repr=False, compare=False) @@ -807,7 +808,7 @@ def __len__(self) -> int: return self._len def __getitem__(self, index: int) -> TaskParameterSet: - return {self.name: ParameterValue(type=self.type, value=self.range[index])} + return {self.name: ParameterValue(type=self.type, value=str(self.range[index]))} def validate_containment(self, params: TaskParameterSet): """Checks if the params restricted to this node are part of the node's range.""" diff --git a/src/openjd/model/_types.py b/src/openjd/model/_types.py index f7b41951..e5f2bd9b 100644 --- a/src/openjd/model/_types.py +++ b/src/openjd/model/_types.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from enum import Enum -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Iterable, Type, Union from pydantic import ConfigDict, BaseModel @@ -247,14 +247,13 @@ class JobCreationMetadata: not be provided values, or kwargs even, for these fields. """ - adds_fields: Optional[Callable[[str, "OpenJDModel", SymbolTable], dict[str, Any]]] = field( + adds_fields: Optional[Callable[["OpenJDModel", SymbolTable], dict[str, Any]]] = field( default=None ) """This property defines a callable that uses the instantiation context (i.e. SymbolTable) and can materialize new fields that are not already present in the model. - arg0 - The model key where this model was found. - arg1 - The model that is adding a value. - arg2 - The symbol table used in the instantiation. + arg0 - The model that is adding a value. + arg1 - The symbol table used in the instantiation. Use-case: Transforming Job Parameters from their Template form to Job form; we inject the value of the parameter from the SymbolTable into the Job. """ @@ -322,3 +321,36 @@ def _check_constraints(self, value: Any) -> None: ValueError if the value does not meet at least one constraint """ pass + + +class ModelParsingContextInterface(ABC): + """Context required while parsing an OpenJDModel. A subclass + must be provided when calling model_validate. + + OpenJDModelSubclass.model_validate(data, context=ModelParsingContext()) + + Individual validators receive this value as ValidationInfo.context. + """ + + spec_rev: SpecificationRevision + """This contains the revision of the Open Job Description being parsed (e.g. "2023-09"). + By providing it in the context, shared code like the FormatString class can do + version-specific processing. + """ + + extensions: set[str] + """When parsing a top-level model instance, this is the set of supported extension names. + The 'extensions' field is second in the list of model properties for both the job template + and environment template, and when that field is processed it becomes the set of extensions + that the template requested. + + When fields of a model that depend on an extension are processed, its validators should + check whether the needed extension is included in the context and adjust its parsing + as written in the specification. + """ + + def __init__( + self, *, spec_rev: SpecificationRevision, supported_extensions: Optional[Iterable[str]] + ) -> None: + self.spec_rev = spec_rev + self.extensions = set(supported_extensions or []) diff --git a/src/openjd/model/v2023_09/_model.py b/src/openjd/model/v2023_09/_model.py index 9b1e4807..ab95fb11 100644 --- a/src/openjd/model/v2023_09/_model.py +++ b/src/openjd/model/v2023_09/_model.py @@ -48,6 +48,7 @@ JobCreateAsMetadata, JobCreationMetadata, JobParameterInterface, + ModelParsingContextInterface, OpenJDModel, ResolutionScope, SpecificationRevision, @@ -56,7 +57,7 @@ ) -class ModelParsingContext: +class ModelParsingContext(ModelParsingContextInterface): """Context required while parsing an OpenJDModel. An instance of this class must be provided when calling model_validate. @@ -65,19 +66,10 @@ class ModelParsingContext: Individual validators receive this value as ValidationInfo.context. """ - extensions: set[str] - """When parsing a top-level model instance, this is the set of supported extension names. - The 'extensions' field is second in the list of model properties for both the job template - and environment template, and when that field is processed it becomes the set of extensions - that the template requested. - - When fields of a model that depend on an extension are processed, its validators should - check whether the needed extension is included in the context and adjust its parsing - as written in the specification. - """ - def __init__(self, *, supported_extensions: Optional[Iterable[str]] = None) -> None: - self.extensions = set(supported_extensions or []) + super().__init__( + spec_rev=SpecificationRevision.v2023_09, supported_extensions=supported_extensions + ) class OpenJDModel_v2023_09(OpenJDModel): # noqa: N801 @@ -91,7 +83,7 @@ def supported_extension_names() -> set[str]: class ExtensionName(str, Enum): - """Enumerant of all extensions supported for the 2023-09 specification revision. + """Enumeration of all extensions supported for the 2023-09 specification revision. This appears in the 'extensions' list property of all model instances. """ @@ -517,7 +509,7 @@ class RangeString(FormatString): StringRangeList = Annotated[list[TaskParameterStringValue], Field(min_length=1, max_length=1024)] TaskParameterStringValueAsJob = Annotated[str, StringConstraints(min_length=0, max_length=1024)] -TaskRangeList = list[TaskParameterStringValueAsJob] +TaskRangeList = list[Union[TaskParameterStringValueAsJob, int, float, Decimal]] # Target model for task parameters when instantiating a job. @@ -529,17 +521,6 @@ class RangeListTaskParameterDefinition(OpenJDModel_v2023_09): # has a value when type is CHUNK[INT], which is only possible from the TASK_CHUNKING extension chunks: Optional[TaskChunksDefinition] = None - @field_validator("range", mode="before") - @classmethod - def _coerce_to_string(cls, value: Any) -> Any: - # Coerce any int, float, or Decimal values into str - def coerce(v: Any) -> Any: - if isinstance(v, (int, float, Decimal)): - return str(v) - return v - - return [coerce(item) for item in value] - class RangeExpressionTaskParameterDefinition(OpenJDModel_v2023_09): # element type of items in the range @@ -565,15 +546,17 @@ class TaskChunksDefinition(OpenJDModel_v2023_09): @field_validator("defaultTaskCount", mode="before") @classmethod - def _validate_default_task_count(cls, value: Any) -> Any: - return validate_int_fmtstring_field(value, ge=1) + def _validate_default_task_count(cls, value: Any, info: ValidationInfo) -> Any: + context = cast(Optional[ModelParsingContextInterface], info.context) + return validate_int_fmtstring_field(value, ge=1, context=context) @field_validator("targetRuntimeSeconds", mode="before") @classmethod - def _validate_target_runtime_seconds(cls, value: Any) -> Any: + def _validate_target_runtime_seconds(cls, value: Any, info: ValidationInfo) -> Any: if value is None: return value - return validate_int_fmtstring_field(value, ge=0) + context = cast(Optional[ModelParsingContextInterface], info.context) + return validate_int_fmtstring_field(value, ge=0, context=context) class IntTaskParameterDefinition(OpenJDModel_v2023_09): @@ -613,14 +596,15 @@ def _get_range_task_param_type(self: Any) -> Type[OpenJDModel]: @field_validator("range", mode="before") @classmethod - def _validate_range_element_type(cls, value: Any) -> Any: + def _validate_range_element_type(cls, value: Any, info: ValidationInfo) -> Any: # pydantic will automatically type coerce values into integers. We explicitly # want to reject non-integer values, so this *pre* validator validates the # value *before* pydantic tries to type coerce it. # We do allow coersion from a string since we want to allow "1", and # "1.2" or "a" will fail the type coersion if isinstance(value, list): - return validate_list_field(value, validate_int_fmtstring_field) + context = cast(Optional[ModelParsingContextInterface], info.context) + return validate_list_field(value, validate_int_fmtstring_field, context=context) elif isinstance(value, RangeString): # Nothing to do - it's guaranteed to be a format string at this point pass @@ -671,12 +655,13 @@ class FloatTaskParameterDefinition(OpenJDModel_v2023_09): @field_validator("range", mode="before") @classmethod - def _validate_range_element_type(cls, value: Any) -> Any: + def _validate_range_element_type(cls, value: Any, info: ValidationInfo) -> Any: # pydantic will automatically type coerce values into floats. We explicitly # want to reject non-integer values, so this *pre* validator validates the # value *before* pydantic tries to type coerce it. if isinstance(value, list): - return validate_list_field(value, validate_float_fmtstring_field) + context = cast(Optional[ModelParsingContextInterface], info.context) + return validate_list_field(value, validate_float_fmtstring_field, context=context) return value @@ -789,14 +774,15 @@ def _validate_task_chunking_extension( @field_validator("range", mode="before") @classmethod - def _validate_range_element_type(cls, value: Any) -> Any: + def _validate_range_element_type(cls, value: Any, info: ValidationInfo) -> Any: # pydantic will automatically type coerce values into integers. We explicitly # want to reject non-integer values, so this *pre* validator validates the # value *before* pydantic tries to type coerce it. # We do allow coersion from a string since we want to allow "1", and # "1.2" or "a" will fail the type coersion if isinstance(value, list): - return validate_list_field(value, validate_int_fmtstring_field) + context = cast(Optional[ModelParsingContextInterface], info.context) + return validate_list_field(value, validate_int_fmtstring_field, context=context) elif isinstance(value, RangeString): # Nothing to do - it's guaranteed to be a format string at this point pass @@ -1168,7 +1154,7 @@ class JobStringParameterDefinition(OpenJDModel_v2023_09, JobParameterInterface): "allowedValues", "default", }, - adds_fields=lambda key, this, symtab: { + adds_fields=lambda this, symtab: { "value": symtab[f"RawParam.{cast(JobStringParameterDefinition,this).name}"] }, ) @@ -1410,7 +1396,7 @@ class JobPathParameterDefinition(OpenJDModel_v2023_09, JobParameterInterface): "allowedValues", "default", }, - adds_fields=lambda key, this, symtab: { + adds_fields=lambda this, symtab: { "value": symtab[f"RawParam.{cast(JobStringParameterDefinition,this).name}"] }, ) @@ -1631,7 +1617,7 @@ class JobIntParameterDefinition(OpenJDModel_v2023_09): "allowedValues", "default", }, - adds_fields=lambda key, this, symtab: { + adds_fields=lambda this, symtab: { "value": symtab[f"RawParam.{cast(JobIntParameterDefinition,this).name}"] }, ) @@ -1877,7 +1863,7 @@ class JobFloatParameterDefinition(OpenJDModel_v2023_09): "allowedValues", "default", }, - adds_fields=lambda key, this, symtab: { + adds_fields=lambda this, symtab: { "value": symtab[f"RawParam.{cast(JobFloatParameterDefinition,this).name}"] }, ) @@ -2060,14 +2046,34 @@ class AmountRequirement(OpenJDModel_v2023_09): min: Optional[Decimal] = None max: Optional[Decimal] = None - @model_validator(mode="before") + @field_validator("name") @classmethod - def validate_concrete_model(cls, values: dict[str, Any]) -> dict[str, Any]: - # Reuse the AmountRequirementTemplate validation. Because all the template - # variables have been substituted, it will now run validation it couldn't - # before. - AmountRequirementTemplate.model_validate(values) - return values + def _validate_name(cls, v: str, info: ValidationInfo) -> str: + validate_amount_capability_name( + capability_name=v, standard_capabilities=_STANDARD_AMOUNT_CAPABILITIES_NAMES + ) + return v + + @field_validator("min") + @classmethod + def _validate_min(cls, v: Optional[Decimal]) -> Optional[Decimal]: + if v is None: + return v + if v < 0: + raise ValueError(f"Value {v} must be zero or greater") + return v + + @field_validator("max") + @classmethod + def _validate_max(cls, v: Optional[Decimal], info: ValidationInfo) -> Optional[Decimal]: + if v is None: + return v + if v <= 0: + raise ValueError("Value must be greater than 0") + v_min = info.data.get("min") + if v_min is not None and v_min > v: + raise ValueError("Value for 'max' must be greater or equal to 'min'") + return v class AmountRequirementTemplate(OpenJDModel_v2023_09): @@ -2090,14 +2096,12 @@ class AmountRequirementTemplate(OpenJDModel_v2023_09): _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=AmountRequirement), - resolve_fields={ - "name", - }, + resolve_fields={"name"}, ) @field_validator("name") @classmethod - def _validate_name(cls, v: str) -> str: + def _validate_name(cls, v: AmountCapabilityName, info: ValidationInfo) -> AmountCapabilityName: validate_amount_capability_name( capability_name=v, standard_capabilities=_STANDARD_AMOUNT_CAPABILITIES_NAMES ) @@ -2147,14 +2151,33 @@ class AttributeRequirement(OpenJDModel_v2023_09): anyOf: Optional[list[str]] = None allOf: Optional[list[str]] = None - @model_validator(mode="before") + @field_validator("name") @classmethod - def validate_concrete_model(cls, values: dict[str, Any]) -> dict[str, Any]: - # Reuse the AttributeRequirementTemplate validation. Because all the template - # variables have been substituted, it will now run validation it couldn't - # before. - AttributeRequirementTemplate.model_validate(values) - return values + def _validate_name(cls, v: str) -> str: + validate_attribute_capability_name( + capability_name=v, standard_capabilities=_STANDARD_ATTRIBUTE_CAPABILITIES_NAMES + ) + return v + + @field_validator("allOf") + @classmethod + def _validate_allof( + cls, v: Optional[AttributeCapabilityList], info: ValidationInfo + ) -> Optional[AttributeCapabilityList]: + if v is None: + return v + AttributeRequirementTemplate._validate_attribute_list(v, info, True) + return v + + @field_validator("anyOf") + @classmethod + def _validate_anyof( + cls, v: Optional[AttributeCapabilityList], info: ValidationInfo + ) -> Optional[AttributeCapabilityList]: + if v is None: + return v + AttributeRequirementTemplate._validate_attribute_list(v, info, False) + return v class AttributeRequirementTemplate(OpenJDModel_v2023_09): @@ -2190,8 +2213,12 @@ def _validate_name(cls, v: str) -> str: @classmethod def _validate_attribute_list( - cls, v: AttributeCapabilityList, info: ValidationInfo, is_allof: bool + cls, + v: Union[list[Union[AttributeCapabilityValue, str]], AttributeCapabilityList], + info: ValidationInfo, + is_allof: bool, ) -> None: + # This function is also called from AttributeRequirement try: capability_name = info.data["name"].lower() except KeyError: @@ -2207,7 +2234,7 @@ def _validate_attribute_list( for item in v: # If it has expressions like "{{ Param.SomeValue }}", will # validate when those values are substituted. - if len(item.expressions) > 0: + if isinstance(item, FormatString) and len(item.expressions) > 0: continue if item not in standard_capability["values"]: raise ValueError( @@ -2217,7 +2244,7 @@ def _validate_attribute_list( for item in v: # If it has expressions like "{{ Param.SomeValue }}", will # validate when those values are substituted. - if len(item.expressions) > 0: + if isinstance(item, FormatString) and len(item.expressions) > 0: continue if not cls._attribute_capability_value_regex.match(item): raise ValueError(f"Value {item} is not a valid attribute capability value.") @@ -2540,12 +2567,14 @@ def _unique_environment_names( return v @classmethod - def _root_template_prevalidator(cls, values: dict[str, Any]) -> dict[str, Any]: + def _root_template_prevalidator( + cls, values: dict[str, Any], context: Optional[ModelParsingContextInterface] + ) -> dict[str, Any]: # The name of this validator is very important. It is specifically looked for # in the _parse_model function to run this validation as a pre-root-validator # without the usual short-circuit of pre-root-validators that pydantic does. errors = prevalidate_model_template_variable_references( - cast(Type[OpenJDModel], cls), values + cast(Type[OpenJDModel], cls), values, context=context ) if errors: raise ValidationError.from_exception_data(cls.__name__, line_errors=errors) @@ -2709,12 +2738,14 @@ def _unique_parameter_names( return v @classmethod - def _root_template_prevalidator(cls, values: dict[str, Any]) -> dict[str, Any]: + def _root_template_prevalidator( + cls, values: dict[str, Any], context: Optional[ModelParsingContextInterface] + ) -> dict[str, Any]: # The name of this validator is very important. It is specifically looked for # in the _parse_model function to run this validation as a pre-root-validator # without the usual short-circuit of pre-root-validators that pydantic does. errors = prevalidate_model_template_variable_references( - cast(Type[OpenJDModel], cls), values + cast(Type[OpenJDModel], cls), values, context=context ) if errors: raise ValidationError.from_exception_data(cls.__name__, line_errors=errors) diff --git a/test/openjd/model/_internal/test_create_job.py b/test/openjd/model/_internal/test_create_job.py index 7160f936..442c9d27 100644 --- a/test/openjd/model/_internal/test_create_job.py +++ b/test/openjd/model/_internal/test_create_job.py @@ -16,6 +16,7 @@ JobCreationMetadata, OpenJDModel, ) +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 class BaseModelForTesting(OpenJDModel): @@ -203,18 +204,24 @@ def test_as_field(self) -> None: # Test that we resolve the desired format strings when they are the value of a field. # GIVEN - f1 = FormatString("{{ Param.V }}") - f2 = FormatString("{{ Param.V2 }}") + f1 = FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()) + f2 = FormatString("{{ Param.V2 }}", context=ModelParsingContext_v2023_09()) + + class ResolvedModel(BaseModelForTesting): + f1: str + f2: str class Model(BaseModelForTesting): f1: FormatString f2: FormatString - _job_creation_metadata = JobCreationMetadata(resolve_fields={"f1"}) + _job_creation_metadata = JobCreationMetadata( + create_as=JobCreateAsMetadata(model=ResolvedModel), resolve_fields={"f1"} + ) model = Model(f1=f1, f2=f2) symtab = SymbolTable(source={"Param.V": "ValueOfV"}) - expected = Model(f1="ValueOfV", f2=f2) + expected = ResolvedModel(f1="ValueOfV", f2=f2) # WHEN result = instantiate_model(model, symtab) @@ -226,18 +233,24 @@ def test_within_list(self) -> None: # Test that we resolve the desired format strings when they are located within a list field. # GIVEN - f1 = FormatString("{{ Param.V }}") - f2 = FormatString("{{ Param.V2 }}") + f1 = FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()) + f2 = FormatString("{{ Param.V2 }}", context=ModelParsingContext_v2023_09()) + + class TargetModel(BaseModelForTesting): + f1: list[str] + f2: FormatString class Model(BaseModelForTesting): f1: list[FormatString] f2: FormatString - _job_creation_metadata = JobCreationMetadata(resolve_fields={"f1"}) + _job_creation_metadata = JobCreationMetadata( + resolve_fields={"f1"}, create_as=JobCreateAsMetadata(model=TargetModel) + ) model = Model(f1=[f1, f1, f1], f2=f2) symtab = SymbolTable(source={"Param.V": "ValueOfV"}) - expected = Model(f1=["ValueOfV", "ValueOfV", "ValueOfV"], f2=f2) + expected = TargetModel(f1=["ValueOfV", "ValueOfV", "ValueOfV"], f2=f2) # WHEN result = instantiate_model(model, symtab) @@ -250,18 +263,24 @@ def test_within_mixed_list(self) -> None: # amongst other values that are not format strings. # GIVEN - f1 = FormatString("{{ Param.V }}") - f2 = FormatString("{{ Param.V2 }}") + f1 = FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()) + f2 = FormatString("{{ Param.V2 }}", context=ModelParsingContext_v2023_09()) + + class TargetModel(BaseModelForTesting): + f1: list[Union[int, str]] + f2: FormatString class Model(BaseModelForTesting): f1: list[Union[int, FormatString]] f2: FormatString - _job_creation_metadata = JobCreationMetadata(resolve_fields={"f1"}) + _job_creation_metadata = JobCreationMetadata( + resolve_fields={"f1"}, create_as=JobCreateAsMetadata(model=TargetModel) + ) model = Model(f1=[f1, 12], f2=f2) symtab = SymbolTable(source={"Param.V": "ValueOfV"}) - expected = Model(f1=["ValueOfV", 12], f2=f2) + expected = TargetModel(f1=["ValueOfV", 12], f2=f2) # WHEN result = instantiate_model(model, symtab) @@ -355,22 +374,20 @@ class TargetModel(BaseModelForTesting): name: str type: str n1: str - n2: str class Model(BaseModelForTesting): name: str type: str _job_creation_metadata = JobCreationMetadata( create_as=JobCreateAsMetadata(model=TargetModel), - adds_fields=lambda key, model, symtab: { - "n1": key, - "n2": symtab[f"Param.{cast(Model, model).name}"], + adds_fields=lambda model, symtab: { + "n1": symtab[f"Param.{cast(Model, model).name}"], }, ) model = Model(name="Foo", type="INT") symtab = SymbolTable(source={"Param.Foo": "FooValue"}) - expected = TargetModel(name="Foo", type="INT", n1="", n2="FooValue") + expected = TargetModel(name="Foo", type="INT", n1="FooValue") # WHEN result = instantiate_model(model, symtab) @@ -466,7 +483,9 @@ class Model(BaseModelForTesting): create_as=JobCreateAsMetadata(model=TargetModel), resolve_fields={"ff"} ) - model = Model(vv=-10, ff="{{ Param.V }}") + model = Model( + vv=-10, ff=FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()) + ) symtab = SymbolTable( source={"Param.V": "{{ Foo.Bar"} # a bad format string to fire an exception ) @@ -507,7 +526,12 @@ class Model(BaseModelForTesting): create_as=JobCreateAsMetadata(model=TargetModel) ) - model = Model(ii={"vv": -10, "ff": "{{ Param.V }}"}) + model = Model( + ii={ + "vv": -10, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + } + ) symtab = SymbolTable( source={"Param.V": "{{ Foo.Bar"} # a bad format string to fire an exception ) @@ -542,7 +566,18 @@ class InnerModel(BaseModelForTesting): class Model(BaseModelForTesting): ii: list[InnerModel] - model = Model(ii=[{"vv": -10, "ff": "{{ Param.V }}"}, {"vv": -5, "ff": "{{ Param.V }}"}]) + model = Model( + ii=[ + { + "vv": -10, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + }, + { + "vv": -5, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + }, + ] + ) symtab = SymbolTable( source={"Param.V": "{{ Foo.Bar"} # a bad format string to fire an exception ) @@ -568,7 +603,7 @@ def test_within_reshape_list(self) -> None: # GIVEN class TargetInner(BaseModelForTesting): vv: PositiveInt - ff: FormatString + ff: int class TargetModel(BaseModelForTesting): ii: dict[str, TargetInner] @@ -592,12 +627,20 @@ class Model(BaseModelForTesting): model = Model( ii=[ - {"name": "foo", "vv": -10, "ff": "{{ Param.V }}"}, - {"name": "bar", "vv": -5, "ff": "{{ Param.V }}"}, + { + "name": "foo", + "vv": -10, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + }, + { + "name": "bar", + "vv": -5, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + }, ] ) symtab = SymbolTable( - source={"Param.V": "{{ Foo.Bar"} # a bad format string to fire an exception + source={"Param.V": "not an integer"} # a bad format string to fire an exception ) # WHEN @@ -620,7 +663,7 @@ def test_within_dict(self) -> None: # GIVEN class TargetInner(BaseModelForTesting): vv: PositiveInt - ff: FormatString + ff: int class InnerModel(BaseModelForTesting): vv: int @@ -634,10 +677,19 @@ class Model(BaseModelForTesting): dd: dict[str, InnerModel] model = Model( - dd={"foo": {"vv": -10, "ff": "{{ Param.V }}"}, "bar": {"vv": -5, "ff": "{{ Param.V }}"}} + dd={ + "foo": { + "vv": -10, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + }, + "bar": { + "vv": -5, + "ff": FormatString("{{ Param.V }}", context=ModelParsingContext_v2023_09()), + }, + } ) symtab = SymbolTable( - source={"Param.V": "{{ Foo.Bar"} # a bad format string to fire an exception + source={"Param.V": "not an integer"} # a non-integer to fire an exception ) # WHEN @@ -647,7 +699,7 @@ class Model(BaseModelForTesting): errors = exc.errors() # THEN - assert len(errors) == 4 + assert len(errors) == 4, str(exc) locs = [err["loc"] for err in errors] assert ("dd", "foo", "vv") in locs assert ("dd", "foo", "ff") in locs diff --git a/test/openjd/model/_internal/test_variable_reference_validation.py b/test/openjd/model/_internal/test_variable_reference_validation.py index 6e1ba958..fa9f161d 100644 --- a/test/openjd/model/_internal/test_variable_reference_validation.py +++ b/test/openjd/model/_internal/test_variable_reference_validation.py @@ -15,6 +15,7 @@ ResolutionScope, TemplateVariableDef, ) +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 # arg2 = Whether a var defined in the arg0 scope is available in the arg1 scope. # TEMPLATE scope -> referenced in only TEMPLATE, SESSION, and TASK scope @@ -66,7 +67,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{ Param.Foo }}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -91,7 +94,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": {"a": "{{ Param.Foo }}"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -125,7 +130,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -160,7 +167,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -186,7 +195,9 @@ class BaseModel(OpenJDModel): data = {"defn": {"name": "Foo"}, "ref": "{{ Param.Foo }}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 1 @@ -208,7 +219,9 @@ class BaseModel(OpenJDModel): data = dict[str, Any]() # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -230,7 +243,9 @@ class BaseModel(OpenJDModel): data = {"name": 12, "ref": "this is okay"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -253,7 +268,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{Param.Foo"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -276,7 +293,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "No variable reference"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -299,7 +318,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{}}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -323,7 +344,9 @@ class BaseModel(OpenJDModel): data = {"lit": "Bob", "name": "Foo", "ref": "{{ Param.Foo }}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -355,7 +378,9 @@ class BaseModel(OpenJDModel): data = {"ref": "{{ Param.Foo }}", "sub": "this is not a dict"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 1 # Bad reference to Param.Foo @@ -387,7 +412,9 @@ class BaseModel(OpenJDModel): data = {"ref": ["{{Param.Foo}}"], "sub": {"name": "Foo"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -409,7 +436,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{ Param.Bar }}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -431,7 +460,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": {12: "{{ Param.Bar }}"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -461,7 +492,9 @@ class BaseModel(OpenJDModel): data = {"sub": {"name": "Foo", "ref": "{{ Root.Param.Foo }}"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -487,7 +520,9 @@ class BaseModel(OpenJDModel): data = {"sub": {"name": "Foo", "ref": "{{ Sub.Foo }}"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -514,7 +549,9 @@ class BaseModel(OpenJDModel): data = {"sub": {"name": "Foo", "ref": "{{ Sub.Inner.Foo }}"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -544,7 +581,9 @@ class BaseModel(OpenJDModel): data = {"sub": {"ref": "{{ Root.Foo }} {{ New.Bar }}"}} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 2) @@ -587,7 +626,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 3) @@ -613,7 +654,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": ["{{ Param.Foo }}", "{{ Param.Foo }}", "{{ Param.Foo }}"]} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 3) @@ -636,7 +679,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{ Param.Foo }}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -654,12 +699,15 @@ class BaseModel(OpenJDModel): _template_variable_definitions = DefinesTemplateVariables( defines={TemplateVariableDef(prefix="|Param.", resolves=ResolutionScope.TEMPLATE)}, field="name", + inject={"|Param.Bar"}, ) data = {"name": "Foo", "ref": ["{{ Param.Foo }}", 12, {"item": "{{Param.Bar}}"}]} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -695,7 +743,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -757,7 +807,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -820,7 +872,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -877,7 +931,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -945,7 +1001,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 2) @@ -1016,7 +1074,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == ( @@ -1078,7 +1138,9 @@ class BaseModel(OpenJDModel): } # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -1114,7 +1176,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{Param.Foo}}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -1142,7 +1206,9 @@ class BaseModel(OpenJDModel): ) # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 @@ -1183,7 +1249,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": "{{Param.Foo}}"} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -1223,7 +1291,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": ["{{Param.Foo}}"]} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == (0 if available else 1) @@ -1254,7 +1324,9 @@ class BaseModel(OpenJDModel): data = {"name": "Foo", "ref": ["{{Param.Foo}}"]} # WHEN - errors = prevalidate_model_template_variable_references(BaseModel, data) + errors = prevalidate_model_template_variable_references( + BaseModel, data, context=ModelParsingContext_v2023_09() + ) # THEN assert len(errors) == 0 diff --git a/test/openjd/model/format_strings/test_dyn_constrained_str.py b/test/openjd/model/format_strings/test_dyn_constrained_str.py index ff324f09..49878667 100644 --- a/test/openjd/model/format_strings/test_dyn_constrained_str.py +++ b/test/openjd/model/format_strings/test_dyn_constrained_str.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ValidationError from openjd.model._format_strings._dyn_constrained_str import DynamicConstrainedStr +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 class TestDyanamicConstrainedStr: @@ -18,7 +19,7 @@ class Model(BaseModel): s: DynamicConstrainedStr # WHEN - Model.model_validate({"s": "123"}) + Model.model_validate({"s": "123"}, context=ModelParsingContext_v2023_09()) # THEN # raised no error @@ -31,7 +32,7 @@ def test_encodes_as_str(self) -> None: class Model(BaseModel): s: DynamicConstrainedStr - model = Model(s="12") + model = Model.model_validate({"s": "12"}, context=ModelParsingContext_v2023_09()) # WHEN as_dict = model.model_dump() @@ -48,7 +49,7 @@ class Model(BaseModel): # WHEN with pytest.raises(ValidationError) as excinfo: - Model.model_validate({"s": 123}) + Model.model_validate({"s": 123}, context=ModelParsingContext_v2023_09()) # THEN assert len(excinfo.value.errors()) == 1 @@ -64,7 +65,7 @@ class Model(BaseModel): s: StrType # WHEN - Model.model_validate({"s": "0" * 10}) + Model.model_validate({"s": "0" * 10}, context=ModelParsingContext_v2023_09()) # THEN # raised no error @@ -81,7 +82,7 @@ class Model(BaseModel): # WHEN with pytest.raises(ValidationError) as excinfo: - Model.model_validate({"s": "0" * 9}) + Model.model_validate({"s": "0" * 9}, context=ModelParsingContext_v2023_09()) # THEN assert len(excinfo.value.errors()) == 1 @@ -97,7 +98,7 @@ class Model(BaseModel): s: StrType # WHEN - Model.model_validate({"s": "0" * 10}) + Model.model_validate({"s": "0" * 10}, context=ModelParsingContext_v2023_09()) # THEN # raised no error @@ -114,7 +115,7 @@ class Model(BaseModel): # WHEN with pytest.raises(ValidationError) as excinfo: - Model.model_validate({"s": "0" * 11}) + Model.model_validate({"s": "0" * 11}, context=ModelParsingContext_v2023_09()) # THEN assert len(excinfo.value.errors()) == 1 @@ -130,7 +131,7 @@ class Model(BaseModel): s: StrType # WHEN - Model.model_validate({"s": "0" * 10}) + Model.model_validate({"s": "0" * 10}, context=ModelParsingContext_v2023_09()) # THEN # no errors raised @@ -147,7 +148,7 @@ class Model(BaseModel): # WHEN with pytest.raises(ValidationError) as excinfo: - Model.model_validate({"s": "1" * 10}) + Model.model_validate({"s": "1" * 10}, context=ModelParsingContext_v2023_09()) # THEN assert len(excinfo.value.errors()) == 1 @@ -163,7 +164,7 @@ class Model(BaseModel): s: StrType # WHEN - Model.model_validate({"s": "0" * 10}) + Model.model_validate({"s": "0" * 10}, context=ModelParsingContext_v2023_09()) # THEN # no errors raised @@ -180,7 +181,39 @@ class Model(BaseModel): # WHEN with pytest.raises(ValidationError) as excinfo: - Model.model_validate({"s": "1" * 10}) + Model.model_validate({"s": "1" * 10}, context=ModelParsingContext_v2023_09()) # THEN assert len(excinfo.value.errors()) == 1 + + def test_get_json_schema(self) -> None: + # GIVEN + class StrType(DynamicConstrainedStr): + _min_length = 10 + _regex = re.compile(r"0+") + + @classmethod + def _max_length(cls): + return 20 + + class Model(BaseModel): + s: StrType + + # WHEN + schema = Model.model_json_schema() + + # THEN + assert schema == { + "title": "Model", + "type": "object", + "properties": { + "s": { + "title": "S", + "type": "string", + "minLength": 10, + "maxLength": 20, + "pattern": "0+", + } + }, + "required": ["s"], + } diff --git a/test/openjd/model/format_strings/test_expression.py b/test/openjd/model/format_strings/test_expression.py index b27d5d29..f4bc5a54 100644 --- a/test/openjd/model/format_strings/test_expression.py +++ b/test/openjd/model/format_strings/test_expression.py @@ -6,14 +6,15 @@ from openjd.model import ExpressionError, SymbolTable, TokenError from openjd.model._format_strings._expression import InterpolationExpression -from openjd.model._format_strings._parser import Parser +from openjd.model._format_strings._parser import FormatStringExprParser_v2023_09 +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 class TestInterpolationExpression: def test_init_builds_expr_tree(self): - with patch.object(Parser, "parse") as mock: + with patch.object(FormatStringExprParser_v2023_09, "parse") as mock: # WHEN - InterpolationExpression("Foo.Bar") + InterpolationExpression("Foo.Bar", context=ModelParsingContext_v2023_09()) # THEN mock.assert_called_once_with("Foo.Bar") @@ -24,7 +25,7 @@ def test_init_reraises_parse_error(self): # THEN with pytest.raises(TokenError): - InterpolationExpression(expr) + InterpolationExpression(expr, context=ModelParsingContext_v2023_09()) def test_init_reraises_tokenizer_error(self): # GIVEN @@ -32,12 +33,12 @@ def test_init_reraises_tokenizer_error(self): # THEN with pytest.raises(TokenError): - InterpolationExpression(expr) + InterpolationExpression(expr, context=ModelParsingContext_v2023_09()) def test_validate_success(self) -> None: # GIVEN symbols = set(("Test.Name",)) - expr = InterpolationExpression("Test.Name") + expr = InterpolationExpression("Test.Name", context=ModelParsingContext_v2023_09()) # THEN expr.validate_symbol_refs(symbols=symbols) # Does not raise @@ -67,7 +68,7 @@ def test_validate_success(self) -> None: ) def test_validate_error(self, symbols: set[str], expr: str, error_matches: str) -> None: # GIVEN - test = InterpolationExpression(expr) + test = InterpolationExpression(expr, context=ModelParsingContext_v2023_09()) # THEN with pytest.raises(ValueError, match=error_matches): @@ -77,7 +78,7 @@ def test_evaluate_success(self): # GIVEN symtab = SymbolTable() symtab["Test.Name"] = "value" - expr = InterpolationExpression("Test.Name") + expr = InterpolationExpression("Test.Name", context=ModelParsingContext_v2023_09()) # WHEN result = expr.evaluate(symtab=symtab) @@ -91,7 +92,7 @@ def test_evaluate_fails(self): symtab["Test.Name"] = "value" # WHEN - expr = InterpolationExpression("Test.Fail") + expr = InterpolationExpression("Test.Fail", context=ModelParsingContext_v2023_09()) # THEN with pytest.raises(ExpressionError) as exc: @@ -103,7 +104,7 @@ def test_evaluate_badtype(self): # GIVEN symtab = SymbolTable() symtab["Test.Name"] = {"foo": "bar"} - expr = InterpolationExpression("Test.Name") + expr = InterpolationExpression("Test.Name", context=ModelParsingContext_v2023_09()) # THEN with pytest.raises(ExpressionError): diff --git a/test/openjd/model/format_strings/test_format_string.py b/test/openjd/model/format_strings/test_format_string.py index 8a8da1c9..a65b6994 100644 --- a/test/openjd/model/format_strings/test_format_string.py +++ b/test/openjd/model/format_strings/test_format_string.py @@ -4,6 +4,7 @@ from typing import Union from openjd.model import SymbolTable +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 from openjd.model._format_strings import FormatString, FormatStringError @@ -12,7 +13,7 @@ def test_original_value(): input = "input" # WHEN - format_string = FormatString(input) + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) # THEN assert format_string.original_value == input @@ -23,7 +24,7 @@ def test_expression_property(): input = "a{{ Test.val }}" # WHEN - format_string = FormatString(input) + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) # THEN assert len(format_string.expressions) == 1 @@ -49,7 +50,7 @@ def test_expression_property(): def test_nonvalid_strings(input): # THEN with pytest.raises(FormatStringError, match="Failed to parse interpolation expression"): - FormatString(input) + FormatString(input, context=ModelParsingContext_v2023_09()) class TestFormatStringResolve: @@ -59,7 +60,7 @@ def test_with_empty_table(self, input: str) -> None: symtab = SymbolTable() # WHEN - format_string = FormatString(input) + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) # THEN assert format_string.resolve(symtab=symtab) == input @@ -77,7 +78,7 @@ def test_with_value(self, input: str, expected: str) -> None: symtab = SymbolTable() # WHEN - format_string = FormatString(input) + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) symtab["Test.val"] = 4 # THEN @@ -100,7 +101,7 @@ def test_multiple_expressions( symtab = SymbolTable() # WHEN - format_string = FormatString(input) + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) symtab["Test.val"] = val symtab["Test.end"] = end @@ -113,7 +114,7 @@ def test_without_entry_in_table(self): symtab = SymbolTable() # WHEN - format_string = FormatString(input) + format_string = FormatString(input, context=ModelParsingContext_v2023_09()) symtab["Test.val"] = 4.098 # THEN diff --git a/test/openjd/model/format_strings/test_parser.py b/test/openjd/model/format_strings/test_parser.py index 3e4b8d8b..6d7ce88f 100644 --- a/test/openjd/model/format_strings/test_parser.py +++ b/test/openjd/model/format_strings/test_parser.py @@ -4,43 +4,29 @@ from openjd.model import ExpressionError, TokenError from openjd.model._format_strings._nodes import FullNameNode -from openjd.model._format_strings._parser import Parser +from openjd.model._format_strings._parser import parse_format_string_expr +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_v2023_09 class TestParser: def test_propagates_error(self): - # GIVEN - parser = Parser() - - # THEN with pytest.raises(TokenError): - parser.parse("!") + parse_format_string_expr("!", context=ModelParsingContext_v2023_09()) @pytest.mark.parametrize("name", ["Foo", "Foo.Bar", "Foo.Bar.Baz", "Foo.Bar.Baz.Wuz"]) def test_parse_names(self, name): - # GIVEN - parser = Parser() - # WHEN - result = parser.parse(name) + result = parse_format_string_expr(name, context=ModelParsingContext_v2023_09()) # THEN assert isinstance(result, FullNameNode) assert result.name == name def test_fails_empty(self): - # GIVEN - parser = Parser() - - # THEN with pytest.raises(ExpressionError): - parser.parse("") + parse_format_string_expr("", context=ModelParsingContext_v2023_09()) @pytest.mark.parametrize("text", [".", "Foo.", "Foo..", "Foo.Bar Foo", "Foo.Bar ."]) def test_fails_nonvalid(self, text): - # GIVEN - parser = Parser() - - # THEN with pytest.raises(ExpressionError): - parser.parse(text) + parse_format_string_expr(text, context=ModelParsingContext_v2023_09()) diff --git a/test/openjd/model/test_capabilities.py b/test/openjd/model/test_capabilities.py index 405bee9e..8d2aba61 100644 --- a/test/openjd/model/test_capabilities.py +++ b/test/openjd/model/test_capabilities.py @@ -5,7 +5,7 @@ from typing import Union from openjd.model import validate_amount_capability_name, validate_attribute_capability_name -from openjd.model.v2023_09 import FormatString +from openjd.model.v2023_09 import FormatString, ModelParsingContext as ModelParsingContext_v2023_09 TEST_BUILTIN_AMOUNTS: list[str] = [ "amount.worker.foo", @@ -32,13 +32,25 @@ def _success_test_values(prefix: str) -> list: pytest.param(f"vendor:{prefix}.custom", id="vendor-defined"), pytest.param(f"{prefix.upper()}.WORKER.FOO", id="caps"), pytest.param(f"VENDOR:{prefix.upper()}.CUSTOM", id="caps vendor"), - pytest.param(FormatString(f"{prefix}.worker.foo"), id="format string no expression"), pytest.param( - FormatString(f"{prefix.upper()}.WORKER.FOO"), id="caps format string no expression" + FormatString(f"{prefix}.worker.foo", context=ModelParsingContext_v2023_09()), + id="format string no expression", ), - pytest.param(FormatString("{{ Param.Foo }}"), id="format string with expression"), pytest.param( - FormatString(f"{prefix}.{{{{ Param.Foo }}}}"), id="format string partial expression" + FormatString( + f"{prefix.upper()}.WORKER.FOO", context=ModelParsingContext_v2023_09() + ), + id="caps format string no expression", + ), + pytest.param( + FormatString("{{ Param.Foo }}", context=ModelParsingContext_v2023_09()), + id="format string with expression", + ), + pytest.param( + FormatString( + f"{prefix}.{{{{ Param.Foo }}}}", context=ModelParsingContext_v2023_09() + ), + id="format string partial expression", ), ] + [ # Test the vendor regex diff --git a/test/openjd/model/test_create_job.py b/test/openjd/model/test_create_job.py index 1519a27a..57b17036 100644 --- a/test/openjd/model/test_create_job.py +++ b/test/openjd/model/test_create_job.py @@ -13,28 +13,22 @@ ParameterValueType, create_job, preprocess_job_parameters, + decode_job_template, + decode_environment_template, ) from openjd.model._parse import _parse_model from openjd.model.v2023_09 import ( - Environment as Environment_2023_09, - EnvironmentTemplate as EnvironmentTemplate_2023_09, Job as Job_2023_09, - JobTemplate as JobTemplate_2023_09, JobParameterType as JobParameterType_2023_09, ) -minimal_job_template_2023_09 = _parse_model( - model=JobTemplate_2023_09, - obj={ - "specificationVersion": "jobtemplate-2023-09", - "name": "name", - "steps": [{"name": "step", "script": {"actions": {"onRun": {"command": "do thing"}}}}], - }, -) -minimal_environment_2023_09 = _parse_model( - model=Environment_2023_09, - obj={"name": "env", "script": {"actions": {"onEnter": {"command": "do a thing"}}}}, -) +minimal_steps_v2023_09 = [ + {"name": "step", "script": {"actions": {"onRun": {"command": "do thing"}}}} +] +minimal_environment_2023_09 = { + "name": "env", + "script": {"actions": {"onEnter": {"command": "do a thing"}}}, +} class TestPreprocessJobParameters_2023_09: # noqa: N801 @@ -63,16 +57,18 @@ def fake_template_dir_and_cwd(): for param_type in JobParameterType_2023_09 ], ) - def test_handles_parameter_type(self, param_type: str) -> None: + def test_preprocess_job_parameters_handles_parameter_type(self, param_type: str) -> None: # Test that we can process all known kinds of parameters # GIVEN job_parameter_values: JobParameterInputValues = {"Foo": "12"} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, - parameterDefinitions=[{"name": "Foo", "type": param_type}], + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "Foo", "type": param_type}], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -105,11 +101,13 @@ def test_handles_parameter_type_without_path_escape_validation(self, param_type: # GIVEN job_parameter_values: JobParameterInputValues = {"Foo": "12"} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, - parameterDefinitions=[{"name": "Foo", "type": param_type}], + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "Foo", "type": param_type}], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -165,11 +163,13 @@ def test_path_parameter_default_cannot_escape( # GIVEN job_parameter_values: JobParameterInputValues = {} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, - parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": escaping_dir}], + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + steps=minimal_steps_v2023_09, + parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": escaping_dir}], + ) ) # WHEN @@ -189,11 +189,13 @@ def test_job_template_dir_must_be_absolute(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, - parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": "defaultValue"}], + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + steps=minimal_steps_v2023_09, + parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": "defaultValue"}], + ) ) # WHEN @@ -224,11 +226,13 @@ def test_path_parameter_default_escape_without_validation(self, escaping_dir: st # GIVEN job_parameter_values: JobParameterInputValues = {} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, - parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": escaping_dir}], + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + steps=minimal_steps_v2023_09, + parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": escaping_dir}], + ) ) # WHEN @@ -264,11 +268,13 @@ def test_path_parameter_default_escape_without_validation_and_empty_paths( # GIVEN job_parameter_values: JobParameterInputValues = {} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, - parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": escaping_dir}], + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + steps=minimal_steps_v2023_09, + parameterDefinitions=[{"name": "Foo", "type": "PATH", "default": escaping_dir}], + ) ) # WHEN @@ -289,10 +295,12 @@ def test_reports_extra(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {"ThisIsUnknown": "value"} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -318,15 +326,19 @@ def test_reports_extra_with_environments(self) -> None: "ThisIsUnknown": "value", "ThisIsKnown": "value", } - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + steps=minimal_steps_v2023_09, + ) ) - env_template = EnvironmentTemplate_2023_09( - specificationVersion="environment-2023-09", - environment=minimal_environment_2023_09, - parameterDefinitions=[{"name": "ThisIsKnown", "type": "STRING"}], + env_template = decode_environment_template( + template=dict( + specificationVersion="environment-2023-09", + environment=minimal_environment_2023_09, + parameterDefinitions=[{"name": "ThisIsKnown", "type": "STRING"}], + ) ) # WHEN @@ -350,11 +362,13 @@ def test_reports_missing(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = dict() - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[{"name": "ThisIsNotDefined", "type": "STRING"}], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "ThisIsNotDefined", "type": "STRING"}], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -374,16 +388,20 @@ def test_reports_missing_with_environments(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = dict() - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[{"name": "ThisIsNotDefined", "type": "STRING"}], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "ThisIsNotDefined", "type": "STRING"}], + steps=minimal_steps_v2023_09, + ) ) - env_template = EnvironmentTemplate_2023_09( - specificationVersion="environment-2023-09", - environment=minimal_environment_2023_09, - parameterDefinitions=[{"name": "ThisIsAlsoMissing", "type": "STRING"}], + env_template = decode_environment_template( + template=dict( + specificationVersion="environment-2023-09", + environment=minimal_environment_2023_09, + parameterDefinitions=[{"name": "ThisIsAlsoMissing", "type": "STRING"}], + ) ) # WHEN @@ -408,14 +426,16 @@ def test_collects_defaults(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[ - {"name": "Foo", "type": "STRING", "default": "defaultValue"}, - {"name": "Bar", "type": "PATH", "default": "defaultPathValue"}, - ], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[ + {"name": "Foo", "type": "STRING", "default": "defaultValue"}, + {"name": "Bar", "type": "PATH", "default": "defaultPathValue"}, + ], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -440,14 +460,16 @@ def test_empty_path_parameter_passthrough(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {"Bar": ""} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[ - {"name": "Foo", "type": "PATH", "default": ""}, - {"name": "Bar", "type": "PATH", "default": "defaultPathValue"}, - ], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[ + {"name": "Foo", "type": "PATH", "default": ""}, + {"name": "Bar", "type": "PATH", "default": "defaultPathValue"}, + ], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -470,16 +492,22 @@ def test_collects_defaults_with_environments(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[{"name": "Foo", "type": "STRING", "default": "defaultValue"}], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "Foo", "type": "STRING", "default": "defaultValue"}], + steps=minimal_steps_v2023_09, + ) ) - env_template = EnvironmentTemplate_2023_09( - specificationVersion="environment-2023-09", - environment=minimal_environment_2023_09, - parameterDefinitions=[{"name": "Bar", "type": "STRING", "default": "alsoDefaultValue"}], + env_template = decode_environment_template( + template=dict( + specificationVersion="environment-2023-09", + environment=minimal_environment_2023_09, + parameterDefinitions=[ + {"name": "Bar", "type": "STRING", "default": "alsoDefaultValue"} + ], + ) ) # WHEN @@ -505,11 +533,13 @@ def test_ignores_defaults(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {"Foo": "FooValue"} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[{"name": "Foo", "type": "STRING", "default": "defaultValue"}], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "Foo", "type": "STRING", "default": "defaultValue"}], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -529,11 +559,13 @@ def test_checks_contraints(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {"Foo": "two"} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[{"name": "Foo", "type": "STRING", "maxLength": 1}], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "Foo", "type": "STRING", "maxLength": 1}], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -554,16 +586,20 @@ def test_checks_contraints_with_environments(self) -> None: # GIVEN job_parameter_values: JobParameterInputValues = {"Foo": "two", "Bar": "one"} - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[{"name": "Foo", "type": "STRING", "maxLength": 1}], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[{"name": "Foo", "type": "STRING", "maxLength": 1}], + steps=minimal_steps_v2023_09, + ) ) - env_template = EnvironmentTemplate_2023_09( - specificationVersion="environment-2023-09", - environment=minimal_environment_2023_09, - parameterDefinitions=[{"name": "Bar", "type": "STRING", "minLength": 5}], + env_template = decode_environment_template( + template=dict( + specificationVersion="environment-2023-09", + environment=minimal_environment_2023_09, + parameterDefinitions=[{"name": "Bar", "type": "STRING", "minLength": 5}], + ) ) # WHEN @@ -590,14 +626,16 @@ def test_collects_multiple_errors(self) -> None: "Bar": "three", # An extra parameter # missing buz } - job_template = JobTemplate_2023_09( - specificationVersion="jobtemplate-2023-09", - name="test", - parameterDefinitions=[ - {"name": "Foo", "type": "STRING", "maxLength": 1}, - {"name": "Buz", "type": "STRING"}, - ], - steps=minimal_job_template_2023_09.steps, + job_template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="test", + parameterDefinitions=[ + {"name": "Foo", "type": "STRING", "maxLength": 1}, + {"name": "Buz", "type": "STRING"}, + ], + steps=minimal_steps_v2023_09, + ) ) # WHEN @@ -622,9 +660,8 @@ def test_collects_multiple_errors(self) -> None: class TestCreateJob_2023_09: def test_success(self) -> None: # GIVEN - job_template = _parse_model( - model=JobTemplate_2023_09, - obj={ + job_template = decode_job_template( + template={ "specificationVersion": "jobtemplate-2023-09", "name": "Job", "parameterDefinitions": [{"name": "Foo", "type": "INT", "minValue": 10}], @@ -653,9 +690,8 @@ def test_success(self) -> None: def test_with_preprocess_error_from_job_template(self) -> None: # GIVEN - job_template = _parse_model( - model=JobTemplate_2023_09, - obj={ + job_template = decode_job_template( + template={ "specificationVersion": "jobtemplate-2023-09", "name": "Job", "parameterDefinitions": [{"name": "Foo", "type": "INT", "minValue": 10}], @@ -675,9 +711,8 @@ def test_with_preprocess_error_from_job_template(self) -> None: def test_with_preprocess_error_from_environment_template(self) -> None: # GIVEN - job_template = _parse_model( - model=JobTemplate_2023_09, - obj={ + job_template = decode_job_template( + template={ "specificationVersion": "jobtemplate-2023-09", "name": "Job", "parameterDefinitions": [{"name": "Foo", "type": "INT"}], @@ -686,9 +721,8 @@ def test_with_preprocess_error_from_environment_template(self) -> None: ], }, ) - env_template = _parse_model( - model=EnvironmentTemplate_2023_09, - obj={ + env_template = decode_environment_template( + template={ "specificationVersion": "environment-2023-09", "parameterDefinitions": [{"name": "Foo", "type": "INT", "minValue": 10}], "environment": { @@ -712,9 +746,8 @@ def test_with_preprocess_error_from_environment_template(self) -> None: def test_fails_to_instantiate(self) -> None: # GIVEN - job_template = _parse_model( - model=JobTemplate_2023_09, - obj={ + job_template = decode_job_template( + template={ "specificationVersion": "jobtemplate-2023-09", "name": "{{Param.Foo}}", "parameterDefinitions": [{"name": "Foo", "type": "STRING"}], @@ -749,9 +782,8 @@ def test_uneven_parameter_space_association(self) -> None: # definitions to know how large each parameter range is. # GIVEN - job_template = _parse_model( - model=JobTemplate_2023_09, - obj={ + job_template = decode_job_template( + template={ "specificationVersion": "jobtemplate-2023-09", "name": "Job", "steps": [ diff --git a/test/openjd/model/v2023_09/test_create.py b/test/openjd/model/v2023_09/test_create.py index 34bad514..859dd660 100644 --- a/test/openjd/model/v2023_09/test_create.py +++ b/test/openjd/model/v2023_09/test_create.py @@ -3,40 +3,34 @@ # Testing for the model metadata annotations that assist in generating a Job from the # Job Template -from openjd.model import ParameterValue, ParameterValueType, create_job +from decimal import Decimal + +from openjd.model import ParameterValue, ParameterValueType, create_job, decode_job_template from openjd.model.v2023_09 import ( Action, AmountRequirement, - AmountRequirementTemplate, AttributeRequirement, - AttributeRequirementTemplate, CancelationMethodNotifyThenTerminate, CancelationMethodTerminate, - ChunkIntTaskParameterDefinition, EmbeddedFileText, Environment, EnvironmentActions, EnvironmentScript, - FloatTaskParameterDefinition, HostRequirements, - HostRequirementsTemplate, - IntTaskParameterDefinition, Job, - JobFloatParameterDefinition, - JobIntParameterDefinition, JobParameter, - JobStringParameterDefinition, - JobTemplate, RangeExpressionTaskParameterDefinition, RangeListTaskParameterDefinition, Step, StepActions, StepParameterSpace, - StepParameterSpaceDefinition, StepScript, - StepTemplate, - StringTaskParameterDefinition, TaskChunksDefinition, + ModelParsingContext as ModelParsingContext_v2023_09, + FormatString, + CommandString, + DataString, + ArgString, ) @@ -54,192 +48,192 @@ def test_v2023_09(self) -> None: # about those here. # GIVEN - extra_kwargs = {"$schema": "blah "} # special snowflake due to field naming - template = JobTemplate( - **extra_kwargs, - specificationVersion="jobtemplate-2023-09", - name="{{ Param.StringParam }}", - description="job description", - jobEnvironments=[ - Environment( - name="JobEnv", - description="desc", - script=EnvironmentScript( - embeddedFiles=[ - EmbeddedFileText( - name="File", - type="TEXT", - data="some data {{ Param.IntParam }}", - filename="filename.txt", - runnable=False, - ) - ], - actions=EnvironmentActions( - onEnter=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], - timeout=10, - cancelation=CancelationMethodTerminate(mode="TERMINATE"), - ), - onExit=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], - timeout=10, - cancelation=CancelationMethodNotifyThenTerminate( - mode="NOTIFY_THEN_TERMINATE", notifyPeriodInSeconds=30 + template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + name="{{ Param.StringParam }}", + description="job description", + jobEnvironments=[ + dict( + name="JobEnv", + description="desc", + script=dict( + embeddedFiles=[ + dict( + name="File", + type="TEXT", + data="some data {{ Param.IntParam }}", + filename="filename.txt", + runnable=False, + ) + ], + actions=dict( + onEnter=dict( + command="{{ Param.IntParam }}", + args=["{{ Param.FloatParam }}"], + timeout=10, + cancelation=dict(mode="TERMINATE"), + ), + onExit=dict( + command="{{ Param.IntParam }}", + args=["{{ Param.FloatParam }}"], + timeout=10, + cancelation=dict( + mode="NOTIFY_THEN_TERMINATE", notifyPeriodInSeconds=30 + ), ), ), ), + ) + ], + parameterDefinitions=[ + dict( + name="StringParam", + type="STRING", + description="desc", + minLength=1, + maxLength=20, + allowedValues=["TheJobName", "TheOtherJobName"], + default="TheOtherJobName", ), - ) - ], - parameterDefinitions=[ - JobStringParameterDefinition( - name="StringParam", - type="STRING", - description="desc", - minLength=1, - maxLength=20, - allowedValues=["TheJobName", "TheOtherJobName"], - default="TheOtherJobName", - ), - JobStringParameterDefinition( - name="AttrCapabilityName", - type="STRING", - description="desc", - minLength=1, - maxLength=20, - default="attr.mycapability", - ), - JobStringParameterDefinition( - name="AmountCapabilityName", - type="STRING", - description="desc", - minLength=1, - maxLength=20, - default="amount.mycapability", - ), - JobIntParameterDefinition( - name="RangeExpressionParam", - type="INT", - description="desc", - minValue=0, - maxValue=100, - allowedValues=[3, 75], - default=75, - ), - JobIntParameterDefinition( - name="IntParam", - type="INT", - description="desc", - minValue=0, - maxValue=100, - allowedValues=[5, 10, 20], - default=20, - ), - JobFloatParameterDefinition( - name="FloatParam", - type="FLOAT", - description="desc", - minValue=0.0, - maxValue=100.5, - allowedValues=[5, 10, "20.0"], - default=20, - ), - ], - steps=[ - StepTemplate( - name="StepName", - description="desc", - stepEnvironments=[ - Environment( - name="StepEnv", - description="desc", - script=EnvironmentScript( - embeddedFiles=[ - EmbeddedFileText( - name="File", - type="TEXT", - data="some data {{ Param.IntParam }}", - filename="filename.txt", - runnable=False, - ) - ], - actions=EnvironmentActions( - onEnter=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], - timeout=10, - cancelation=CancelationMethodTerminate(mode="TERMINATE"), - ), - onExit=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], - timeout=10, - cancelation=CancelationMethodNotifyThenTerminate( - mode="NOTIFY_THEN_TERMINATE", notifyPeriodInSeconds=30 + dict( + name="AttrCapabilityName", + type="STRING", + description="desc", + minLength=1, + maxLength=20, + default="attr.mycapability", + ), + dict( + name="AmountCapabilityName", + type="STRING", + description="desc", + minLength=1, + maxLength=20, + default="amount.mycapability", + ), + dict( + name="RangeExpressionParam", + type="INT", + description="desc", + minValue=0, + maxValue=100, + allowedValues=[3, 75], + default=75, + ), + dict( + name="IntParam", + type="INT", + description="desc", + minValue=0, + maxValue=100, + allowedValues=[5, 10, 20], + default=20, + ), + dict( + name="FloatParam", + type="FLOAT", + description="desc", + minValue=0.0, + maxValue=100.5, + allowedValues=[5, 10, "20.0"], + default=20, + ), + ], + steps=[ + dict( + name="StepName", + description="desc", + stepEnvironments=[ + dict( + name="StepEnv", + description="desc", + script=dict( + embeddedFiles=[ + dict( + name="File", + type="TEXT", + data="some data {{ Param.IntParam }}", + filename="filename.txt", + runnable=False, + ) + ], + actions=dict( + onEnter=dict( + command="{{ Param.IntParam }}", + args=["{{ Param.FloatParam }}"], + timeout=10, + cancelation=dict(mode="TERMINATE"), + ), + onExit=dict( + command="{{ Param.IntParam }}", + args=["{{ Param.FloatParam }}"], + timeout=10, + cancelation=dict( + mode="NOTIFY_THEN_TERMINATE", + notifyPeriodInSeconds=30, + ), ), ), ), ), - ), - ], - parameterSpace=StepParameterSpaceDefinition( - taskParameterDefinitions=[ - IntTaskParameterDefinition( - name="ParamE", - type="INT", - range="2 - {{ Param.RangeExpressionParam }}", - ), - IntTaskParameterDefinition( - name="ParamI", type="INT", range=[0, "{{ Param.IntParam }}"] - ), - FloatTaskParameterDefinition( - name="ParamF", type="FLOAT", range=[1.1, "{{ Param.FloatParam }}"] - ), - StringTaskParameterDefinition( - name="ParamS", - type="STRING", - range=["foo", "{{ Param.StringParam }}"], - ), ], - combination="ParamS * ParamF * ParamI * ParamE", - ), - script=StepScript( - embeddedFiles=[ - EmbeddedFileText( - name="File", - type="TEXT", - data="some data {{ Param.IntParam }}", - filename="filename.txt", - runnable=False, - ) - ], - actions=StepActions( - onRun=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], - timeout=10, - cancelation=CancelationMethodTerminate(mode="TERMINATE"), - ) + parameterSpace=dict( + taskParameterDefinitions=[ + dict( + name="ParamE", + type="INT", + range="2 - {{ Param.RangeExpressionParam }}", + ), + dict(name="ParamI", type="INT", range=[0, "{{ Param.IntParam }}"]), + dict( + name="ParamF", + type="FLOAT", + range=[1.1, "{{ Param.FloatParam }}"], + ), + dict( + name="ParamS", + type="STRING", + range=["foo", "{{ Param.StringParam }}"], + ), + ], + combination="ParamS * ParamF * ParamI * ParamE", ), - ), - hostRequirements=HostRequirementsTemplate( - amounts=[ - AmountRequirementTemplate(name="amount.worker.vcpu", min=3, max=8), - AmountRequirementTemplate(name="{{Param.AmountCapabilityName}}", min=2), - ], - attributes=[ - AttributeRequirementTemplate( - name="attr.worker.os.family", anyOf=["linux"] - ), - AttributeRequirementTemplate( - name="{{Param.AttrCapabilityName}}", allOf=["{{Param.StringParam}}"] + script=dict( + embeddedFiles=[ + dict( + name="File", + type="TEXT", + data="some data {{ Param.IntParam }}", + filename="filename.txt", + runnable=False, + ) + ], + actions=dict( + onRun=dict( + command="{{ Param.IntParam }}", + args=["{{ Param.FloatParam }}"], + timeout=10, + cancelation=dict(mode="TERMINATE"), + ) ), - ], - ), - ) - ], + ), + hostRequirements=dict( + amounts=[ + dict(name="amount.worker.vcpu", min=3, max=8), + dict(name="{{Param.AmountCapabilityName}}", min=2), + ], + attributes=[ + dict(name="attr.worker.os.family", anyOf=["linux"]), + dict( + name="{{Param.AttrCapabilityName}}", + allOf=["{{Param.StringParam}}"], + ), + ], + ), + ) + ], + ) ) job_parameter_values = { "IntParam": ParameterValue(type=ParameterValueType.INT, value="10"), @@ -264,21 +258,38 @@ def test_v2023_09(self) -> None: EmbeddedFileText( name="File", type="TEXT", - data="some data {{ Param.IntParam }}", + data=DataString( + "some data {{ Param.IntParam }}", + context=ModelParsingContext_v2023_09(), + ), filename="filename.txt", runnable=False, ) ], actions=EnvironmentActions( onEnter=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], + command=CommandString( + "{{ Param.IntParam }}", context=ModelParsingContext_v2023_09() + ), + args=[ + ArgString( + "{{ Param.FloatParam }}", + context=ModelParsingContext_v2023_09(), + ) + ], timeout=10, cancelation=CancelationMethodTerminate(mode="TERMINATE"), ), onExit=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], + command=CommandString( + "{{ Param.IntParam }}", context=ModelParsingContext_v2023_09() + ), + args=[ + ArgString( + "{{ Param.FloatParam }}", + context=ModelParsingContext_v2023_09(), + ) + ], timeout=10, cancelation=CancelationMethodNotifyThenTerminate( mode="NOTIFY_THEN_TERMINATE", notifyPeriodInSeconds=30 @@ -315,21 +326,40 @@ def test_v2023_09(self) -> None: EmbeddedFileText( name="File", type="TEXT", - data="some data {{ Param.IntParam }}", + data=DataString( + "some data {{ Param.IntParam }}", + context=ModelParsingContext_v2023_09(), + ), filename="filename.txt", runnable=False, ) ], actions=EnvironmentActions( onEnter=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], + command=CommandString( + "{{ Param.IntParam }}", + context=ModelParsingContext_v2023_09(), + ), + args=[ + ArgString( + "{{ Param.FloatParam }}", + context=ModelParsingContext_v2023_09(), + ) + ], timeout=10, cancelation=CancelationMethodTerminate(mode="TERMINATE"), ), onExit=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], + command=CommandString( + "{{ Param.IntParam }}", + context=ModelParsingContext_v2023_09(), + ), + args=[ + ArgString( + "{{ Param.FloatParam }}", + context=ModelParsingContext_v2023_09(), + ) + ], timeout=10, cancelation=CancelationMethodNotifyThenTerminate( mode="NOTIFY_THEN_TERMINATE", notifyPeriodInSeconds=30 @@ -344,11 +374,9 @@ def test_v2023_09(self) -> None: "ParamE": RangeExpressionTaskParameterDefinition( type="INT", range="2 - 3" ), - "ParamI": RangeListTaskParameterDefinition( - type="INT", range=["0", "10"] - ), + "ParamI": RangeListTaskParameterDefinition(type="INT", range=[0, "10"]), "ParamF": RangeListTaskParameterDefinition( - type="FLOAT", range=["1.1", "10"] + type="FLOAT", range=[Decimal("1.1"), "10"] ), "ParamS": RangeListTaskParameterDefinition( type="STRING", range=["foo", "TheOtherJobName"] @@ -361,15 +389,25 @@ def test_v2023_09(self) -> None: EmbeddedFileText( name="File", type="TEXT", - data="some data {{ Param.IntParam }}", + data=DataString( + "some data {{ Param.IntParam }}", + context=ModelParsingContext_v2023_09(), + ), filename="filename.txt", runnable=False, ) ], actions=StepActions( onRun=Action( - command="{{ Param.IntParam }}", - args=["{{ Param.FloatParam }}"], + command=CommandString( + "{{ Param.IntParam }}", context=ModelParsingContext_v2023_09() + ), + args=[ + ArgString( + "{{ Param.FloatParam }}", + context=ModelParsingContext_v2023_09(), + ) + ], timeout=10, cancelation=CancelationMethodTerminate(mode="TERMINATE"), ) @@ -409,59 +447,60 @@ def test_v2023_09_extension_task_chunking(self) -> None: # about those here. # GIVEN - extra_kwargs = {"$schema": "blah "} # special snowflake due to field naming - template = JobTemplate( - **extra_kwargs, - specificationVersion="jobtemplate-2023-09", - extensions=["TASK_CHUNKING"], - name="Job {{ Param.IntParam }}", - parameterDefinitions=[ - JobIntParameterDefinition( - name="RangeExpressionParam", - type="INT", - description="desc", - minValue=0, - maxValue=100, - allowedValues=[3, 75], - default=75, - ), - JobIntParameterDefinition( - name="IntParam", - type="INT", - description="desc", - minValue=0, - maxValue=100, - allowedValues=[5, 10, 20], - default=20, - ), - ], - steps=[ - StepTemplate( - name="StepName", - parameterSpace=StepParameterSpaceDefinition( - taskParameterDefinitions=[ - ChunkIntTaskParameterDefinition( - name="ParamE", - type="CHUNK[INT]", - range="2 - {{ Param.RangeExpressionParam }}", - chunks=TaskChunksDefinition( - defaultTaskCount="{{Param.RangeExpressionParam}}", - targetRuntimeSeconds="{{Param.IntParam}}", - rangeConstraint="CONTIGUOUS", + template = decode_job_template( + template=dict( + specificationVersion="jobtemplate-2023-09", + extensions=["TASK_CHUNKING"], + name="Job {{ Param.IntParam }}", + parameterDefinitions=[ + dict( + name="RangeExpressionParam", + type="INT", + description="desc", + minValue=0, + maxValue=100, + allowedValues=[3, 75], + default=75, + ), + dict( + name="IntParam", + type="INT", + description="desc", + minValue=0, + maxValue=100, + allowedValues=[5, 10, 20], + default=20, + ), + ], + steps=[ + dict( + name="StepName", + parameterSpace=dict( + taskParameterDefinitions=[ + dict( + name="ParamE", + type="CHUNK[INT]", + range="2 - {{ Param.RangeExpressionParam }}", + chunks=dict( + defaultTaskCount="{{Param.RangeExpressionParam}}", + targetRuntimeSeconds="{{Param.IntParam}}", + rangeConstraint="CONTIGUOUS", + ), ), + ], + combination="ParamE", + ), + script=dict( + actions=dict( + onRun=dict( + command="{{ Param.IntParam }}", + ) ), - ], - combination="ParamE", - ), - script=StepScript( - actions=StepActions( - onRun=Action( - command="{{ Param.IntParam }}", - ) ), - ), - ) - ], + ) + ], + ), + supported_extensions=["TASK_CHUNKING"], ) job_parameter_values = { "IntParam": ParameterValue(type=ParameterValueType.INT, value="10"), @@ -488,8 +527,12 @@ def test_v2023_09_extension_task_chunking(self) -> None: type="CHUNK[INT]", range="2 - 3", chunks=TaskChunksDefinition( - defaultTaskCount="3", - targetRuntimeSeconds="10", + defaultTaskCount=FormatString( + "3", context=ModelParsingContext_v2023_09() + ), + targetRuntimeSeconds=FormatString( + "10", context=ModelParsingContext_v2023_09() + ), rangeConstraint="CONTIGUOUS", ), ), @@ -499,7 +542,9 @@ def test_v2023_09_extension_task_chunking(self) -> None: script=StepScript( actions=StepActions( onRun=Action( - command="{{ Param.IntParam }}", + command=CommandString( + "{{ Param.IntParam }}", context=ModelParsingContext_v2023_09() + ), ) ), ), diff --git a/test/openjd/model/v2023_09/test_strings.py b/test/openjd/model/v2023_09/test_strings.py index 984fa863..6cabd95a 100644 --- a/test/openjd/model/v2023_09/test_strings.py +++ b/test/openjd/model/v2023_09/test_strings.py @@ -19,6 +19,7 @@ Identifier, JobName, JobTemplateName, + ModelParsingContext, ParameterStringValue, StepName, TaskParameterStringValueAsJob, @@ -105,7 +106,7 @@ def test_parse_success(self, value: str) -> None: data = {"name": value} # WHEN - JobTemplateNameModel.model_validate(data) + JobTemplateNameModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -119,7 +120,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - JobTemplateNameModel.model_validate(data) + JobTemplateNameModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -139,7 +140,7 @@ def test_parse_success(self, value: str) -> None: data = {"name": value} # WHEN - JobNameModel.model_validate(data) + JobNameModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -163,7 +164,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - JobNameModel.model_validate(data) + JobNameModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -183,7 +184,7 @@ def test_parse_success(self, value: str) -> None: data = {"name": value} # WHEN - StepNameModel.model_validate(data) + StepNameModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -207,7 +208,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - StepNameModel.model_validate(data) + StepNameModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -227,7 +228,7 @@ def test_parse_success(self, value: str) -> None: data = {"name": value} # WHEN - EnvironmentNameModel.model_validate(data) + EnvironmentNameModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -251,7 +252,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - EnvironmentNameModel.model_validate(data) + EnvironmentNameModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -280,7 +281,7 @@ def test_parse_success(self, value: str) -> None: data = {"name": value} # WHEN - EnvironmentVariableNameStringModel.model_validate(data) + EnvironmentVariableNameStringModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -315,7 +316,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - EnvironmentVariableNameStringModel.model_validate(data) + EnvironmentVariableNameStringModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -335,7 +336,7 @@ def test_parse_success(self, value: str) -> None: data = {"value": value} # WHEN - EnvironmentVariableValueStringModel.model_validate(data) + EnvironmentVariableValueStringModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -351,7 +352,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - EnvironmentVariableValueStringModel.model_validate(data) + EnvironmentVariableValueStringModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -379,7 +380,7 @@ def test_parse_success(self, value: str) -> None: data = {"id": value} # WHEN - IdentifierModel.model_validate(data) + IdentifierModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -414,7 +415,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - IdentifierModel.model_validate(data) + IdentifierModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -437,7 +438,7 @@ def test_parse_success(self, value: str) -> None: data = {"desc": value} # WHEN - DescriptionModel.model_validate(data) + DescriptionModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -460,7 +461,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - DescriptionModel.model_validate(data) + DescriptionModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -479,7 +480,7 @@ def test_parse_success(self, value: str) -> None: data = {"str": value} # WHEN - ParameterStringModel.model_validate(data) + ParameterStringModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -496,7 +497,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - ParameterStringModel.model_validate(data) + ParameterStringModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -519,7 +520,7 @@ def test_parse_success(self, value: str) -> None: data = {"arg": value} # WHEN - ArgStringModel.model_validate(data) + ArgStringModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -542,7 +543,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - ArgStringModel.model_validate(data) + ArgStringModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -565,7 +566,7 @@ def test_parse_success(self, value: str) -> None: data = {"cmd": value} # WHEN - CommandStringModel.model_validate(data) + CommandStringModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -589,7 +590,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - CommandStringModel.model_validate(data) + CommandStringModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -611,7 +612,7 @@ def test_parse_success(self, value: str) -> None: data = {"expr": value} # WHEN - CombinationExprModel.model_validate(data) + CombinationExprModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -639,7 +640,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - CombinationExprModel.model_validate(data) + CombinationExprModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -654,7 +655,7 @@ def test_parse_success(self, value: str) -> None: data = {"str": value} # WHEN - TaskParameterStringValueAsJobModel.model_validate(data) + TaskParameterStringValueAsJobModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -668,7 +669,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - TaskParameterStringValueAsJobModel.model_validate(data) + TaskParameterStringValueAsJobModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -685,7 +686,7 @@ def test_parse_success(self, value: str) -> None: data = {"str": value} # WHEN - AmountCapabilityNameModel.model_validate(data) + AmountCapabilityNameModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -702,7 +703,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - AmountCapabilityNameModel.model_validate(data) + AmountCapabilityNameModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -719,7 +720,7 @@ def test_parse_success(self, value: str) -> None: data = {"str": value} # WHEN - AttributeCapabilityNameModel.model_validate(data) + AttributeCapabilityNameModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -736,7 +737,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - AttributeCapabilityNameModel.model_validate(data) + AttributeCapabilityNameModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -756,7 +757,7 @@ def test_parse_success(self, value: str) -> None: data = {"str": value} # WHEN - UserInterfaceLabelStringValueModel.model_validate(data) + UserInterfaceLabelStringValueModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -780,7 +781,7 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - UserInterfaceLabelStringValueModel.model_validate(data) + UserInterfaceLabelStringValueModel.model_validate(data, context=ModelParsingContext()) # THEN assert len(excinfo.value.errors()) > 0 @@ -801,7 +802,7 @@ def test_parse_success(self, value: str) -> None: data = {"str": value} # WHEN - FileDialogFilterPatternStringValueModel.model_validate(data) + FileDialogFilterPatternStringValueModel.model_validate(data, context=ModelParsingContext()) # THEN # no exceptions raised @@ -854,7 +855,9 @@ def test_parse_fails(self, data: dict[str, Any]) -> None: # WHEN with pytest.raises(ValidationError) as excinfo: - FileDialogFilterPatternStringValueModel.model_validate(data) + FileDialogFilterPatternStringValueModel.model_validate( + data, context=ModelParsingContext() + ) # THEN assert len(excinfo.value.errors()) > 0 diff --git a/test/openjd/model/v2023_09/test_template_variables.py b/test/openjd/model/v2023_09/test_template_variables.py index 40f89c69..d01133f5 100644 --- a/test/openjd/model/v2023_09/test_template_variables.py +++ b/test/openjd/model/v2023_09/test_template_variables.py @@ -6,7 +6,7 @@ from pydantic import ValidationError from openjd.model._parse import _parse_model -from openjd.model.v2023_09 import JobTemplate +from openjd.model.v2023_09 import JobTemplate, ModelParsingContext as ModelParsingContext_v2023_09 # Some minimal data to reference in tests. ENV_SCRIPT_FOO = { @@ -43,1110 +43,1104 @@ def make_script(env_or_task: str, scriptname: str, scriptdata: str) -> dict: } -class TestJobTemplateVars: - @pytest.mark.parametrize( - "data", - ( - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [STEP_TEMPLATE_FOO], - }, - id="minimum int parameter", - ), - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_FLOAT], - "steps": [STEP_TEMPLATE_FOO], - }, - id="minimum float parameter", - ), - pytest.param( +PARAMETRIZE_CASES: tuple = ( + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [STEP_TEMPLATE_FOO], + }, + id="minimum int parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_FLOAT], + "steps": [STEP_TEMPLATE_FOO], + }, + id="minimum float parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [STEP_TEMPLATE_FOO], + }, + id="minimum string parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{RawParam.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_PATH], + "steps": [STEP_TEMPLATE_FOO], + }, + id="minimum path parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [STEP_TEMPLATE_FOO], + "description": "some text {{Param.NotSubstituted}}", + }, + id="with description", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [STEP_TEMPLATE_FOO], + "jobEnvironments": [ENVIRONMENT_FOO], + }, + id="with environments", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_PATH], + "steps": [STEP_TEMPLATE], + "jobEnvironments": [ENVIRONMENT_FOO], + }, + id="path in environments", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ + STEP_TEMPLATE_FOO, { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [STEP_TEMPLATE_FOO], + "name": "BarStep", + "script": make_script("Task", "BarScript", "echo {{Param.Foo}}"), }, - id="minimum string parameter", - ), - pytest.param( { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{RawParam.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_PATH], - "steps": [STEP_TEMPLATE_FOO], + "name": "BazStep", + "script": make_script( + "Task", "BazScript", "echo {{Param.Foo}} {{Task.File.BazScript}}" + ), }, - id="minimum path parameter", - ), - pytest.param( + ], + }, + id="multiple steps", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [STEP_TEMPLATE_FOO], + "jobEnvironments": [ + ENVIRONMENT_FOO, { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [STEP_TEMPLATE_FOO], - "description": "some text {{Param.NotSubstituted}}", + "name": "BarEnv", + "script": make_script( + "Env", "BarScript", "echo {{Param.Foo}} {{Env.File.BarScript}}" + ), }, - id="with description", - ), - pytest.param( { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [STEP_TEMPLATE_FOO], - "jobEnvironments": [ENVIRONMENT_FOO], + "name": "BazEnv", + "script": make_script( + "Env", + "BazScript", + "echo {{Param.Foo}} {{Session.WorkingDirectory}}", + ), }, - id="with environments", - ), - pytest.param( + ], + }, + id="multiple environments", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_PATH], - "steps": [STEP_TEMPLATE], - "jobEnvironments": [ENVIRONMENT_FOO], + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.RawParam.Foo}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]} + ] + }, }, - id="path in environments", - ), - pytest.param( + ], + }, + id="step with parameter space", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_PATH], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ - STEP_TEMPLATE_FOO, - { - "name": "BarStep", - "script": make_script("Task", "BarScript", "echo {{Param.Foo}}"), - }, - { - "name": "BazStep", - "script": make_script( - "Task", "BazScript", "echo {{Param.Foo}} {{Task.File.BazScript}}" - ), - }, - ], + "name": "BarStep", + "script": make_script( + "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" + ), + "parameterSpace": { + "taskParameterDefinitions": [ + { + "name": "Foo", + "type": "STRING", + "range": ["blah", "{{RawParam.Foo}}"], + } + ] + }, }, - id="multiple steps", - ), - pytest.param( + ], + }, + id="raw path in parameter space", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [STEP_TEMPLATE_FOO], - "jobEnvironments": [ - ENVIRONMENT_FOO, + "name": "BarStep", + "script": make_script( + "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]} + ] + }, + "stepEnvironments": [ { "name": "BarEnv", - "script": make_script( - "Env", "BarScript", "echo {{Param.Foo}} {{Env.File.BarScript}}" - ), - }, - { - "name": "BazEnv", "script": make_script( "Env", - "BazScript", - "echo {{Param.Foo}} {{Session.WorkingDirectory}}", + "BarScript", + "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", ), - }, + } ], }, - id="multiple environments", - ), - pytest.param( + ], + }, + id="step with parameter space and environment", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]}, + {"name": "Bar", "type": "FLOAT", "range": [1, 2]}, + { + "name": "Baz", + "type": "STRING", + "range": ["{{Param.Foo}}"], + }, + ] + }, + "stepEnvironments": [ { - "name": "BarStep", + "name": "BarEnv", "script": make_script( - "Task", + "Env", "BarScript", - "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.RawParam.Foo}}", + "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]} - ] - }, - }, + } ], }, - id="step with parameter space", - ), - pytest.param( + ], + }, + id="string task param can see the job params", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_PATH], - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" - ), - "parameterSpace": { - "taskParameterDefinitions": [ - { - "name": "Foo", - "type": "STRING", - "range": ["blah", "{{RawParam.Foo}}"], - } - ] + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo Hi", + ), + "hostRequirements": { + "amounts": [ + { + "name": "amount.{{Param.Foo}}", + "min": 3, }, - }, - ], - }, - id="raw path in parameter space", - ), - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" - ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]} - ] + ], + "attributes": [ + { + "name": "attr.{{Param.Foo}}", + "anyOf": ["{{Param.Foo}}"], }, - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, - ], - }, - id="step with parameter space and environment", - ), - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", - "BarScript", - "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}}", - ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]}, - {"name": "Bar", "type": "FLOAT", "range": [1, 2]}, - { - "name": "Baz", - "type": "STRING", - "range": ["{{Param.Foo}}"], - }, - ] + { + "name": "attr.allof", + "allOf": ["{{Param.Foo}}"], }, - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, - ], + ], + }, }, - id="string task param can see the job params", - ), - pytest.param( + ], + }, + id="capabilities can see the job params", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo {{Session.WorkingDirectory}} {{Session.HasPathMappingRules}} {{Session.PathMappingRulesFile}}", + ), + "stepEnvironments": [ { - "name": "BarStep", + "name": "BarEnv", "script": make_script( - "Task", + "Env", "BarScript", - "echo Hi", + "echo {{Session.WorkingDirectory}} {{Session.HasPathMappingRules}} {{Session.PathMappingRulesFile}}", ), - "hostRequirements": { - "amounts": [ - { - "name": "amount.{{Param.Foo}}", - "min": 3, - }, - ], - "attributes": [ - { - "name": "attr.{{Param.Foo}}", - "anyOf": ["{{Param.Foo}}"], - }, - { - "name": "attr.allof", - "allOf": ["{{Param.Foo}}"], - }, - ], - }, - }, + } ], }, - id="capabilities can see the job params", - ), - pytest.param( + ], + }, + id="Can reference Session values", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", - "BarScript", - "echo {{Session.WorkingDirectory}} {{Session.HasPathMappingRules}} {{Session.PathMappingRulesFile}}", - ), - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Session.WorkingDirectory}} {{Session.HasPathMappingRules}} {{Session.PathMappingRulesFile}}", - ), - } - ], + "name": "BarStep", + "script": { + "embeddedFiles": [ + { + "name": "file1", + "type": "TEXT", + "data": "{{Task.File.file1}} {{Task.File.file2}}", + }, + { + "name": "file2", + "type": "TEXT", + "data": "{{Task.File.file1}} {{Task.File.file2}}", + }, + ], + "actions": { + "onRun": {"command": "{{Task.File.file1}} {{Task.File.file2}}"} }, - ], + }, }, - id="Can reference Session values", - ), - pytest.param( + ], + }, + id="embedded stepscript files can reference themself and others.", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "jobEnvironments": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [ - { - "name": "BarStep", - "script": { - "embeddedFiles": [ - { - "name": "file1", - "type": "TEXT", - "data": "{{Task.File.file1}} {{Task.File.file2}}", - }, - { - "name": "file2", - "type": "TEXT", - "data": "{{Task.File.file1}} {{Task.File.file2}}", - }, - ], - "actions": { - "onRun": {"command": "{{Task.File.file1}} {{Task.File.file2}}"} - }, + "name": "JobEnv", + "script": { + "embeddedFiles": [ + { + "name": "file1", + "type": "TEXT", + "data": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}", + }, + { + "name": "file2", + "type": "TEXT", + "data": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}", + }, + ], + "actions": { + "onEnter": { + "command": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}" + }, + "onExit": { + "command": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}" }, }, - ], - }, - id="embedded stepscript files can reference themself and others.", - ), - pytest.param( + }, + } + ], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "jobEnvironments": [ + "name": "BarStep", + "stepEnvironments": [ { - "name": "JobEnv", + "name": "StepEnv", "script": { "embeddedFiles": [ { - "name": "file1", + "name": "fileA", "type": "TEXT", - "data": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}", + "data": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}", }, { - "name": "file2", + "name": "fileB", "type": "TEXT", - "data": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}", + "data": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}", }, ], "actions": { "onEnter": { - "command": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}" + "command": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}" }, "onExit": { - "command": "{{Env.File.file1}} {{Env.File.file2}} {{Session.WorkingDirectory}}" + "command": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}" }, }, }, } ], - "steps": [ - { - "name": "BarStep", - "stepEnvironments": [ - { - "name": "StepEnv", - "script": { - "embeddedFiles": [ - { - "name": "fileA", - "type": "TEXT", - "data": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}", - }, - { - "name": "fileB", - "type": "TEXT", - "data": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}", - }, - ], - "actions": { - "onEnter": { - "command": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}" - }, - "onExit": { - "command": "{{Env.File.fileA}} {{Env.File.fileB}} {{Session.WorkingDirectory}}" - }, - }, - }, - } - ], - "script": { - "actions": {"onRun": {"command": "foo"}}, - }, - }, - ], + "script": { + "actions": {"onRun": {"command": "foo"}}, + }, }, - id="embedded env script files can reference themself and others.", - ), - pytest.param( + ], + }, + id="embedded env script files can reference themself and others.", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [STEP_TEMPLATE_FOO], + "jobEnvironments": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [STEP_TEMPLATE_FOO], - "jobEnvironments": [ - { - "name": "VariableEnv", - "variables": { - "VAR_NAME": "{{ Param.Foo }}", - }, - } - ], + "name": "VariableEnv", + "variables": { + "VAR_NAME": "{{ Param.Foo }}", + }, } - ), - pytest.param( + ], + } + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" - ), - "parameterSpace": { - "taskParameterDefinitions": [ - { - "name": "Foo", - "type": "INT", - "range": "1 - {{Param.Foo}}", - } - ] - }, - }, - ], + "name": "BarStep", + "script": make_script( + "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" + ), + "parameterSpace": { + "taskParameterDefinitions": [ + { + "name": "Foo", + "type": "INT", + "range": "1 - {{Param.Foo}}", + } + ] + }, }, - id="int range expression can see params", - ), - ), - ) - def test_parse_success(self, data: dict[str, Any]) -> None: - # Parsing tests of valid Open Job Description JobTemplate - # It is sufficient to check that parsing the input does not - # raise an exception. We trust the Pydantic package's testing - # so, if the input parses then our JobTemplate model is correctly - # constructed for valid input. + ], + }, + id="int range expression can see params", + ), +) - # WHEN - _parse_model(model=JobTemplate, obj=data) - # THEN - # no exception was raised. +@pytest.mark.parametrize( + "data", + PARAMETRIZE_CASES, +) +def test_job_template_variables_parse_success(data: dict[str, Any]) -> None: + # Parsing tests of valid Open Job Description JobTemplate + # It is sufficient to check that parsing the input does not + # raise an exception. We trust the Pydantic package's testing + # so, if the input parses then our JobTemplate model is correctly + # constructed for valid input. - @pytest.mark.parametrize( - "data,error_count", - ( - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Session.WorkingDirectory}}", - "steps": [STEP_TEMPLATE], - }, - 1, - id="session working directory not in 'name' scope", - ), - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_PATH], - "steps": [STEP_TEMPLATE], - }, - 1, - id="path parameter not in 'name' scope", - ), - pytest.param( + # WHEN + _parse_model(model=JobTemplate, obj=data, context=ModelParsingContext_v2023_09()) + + # THEN + # no exception was raised. + + +PARAMETRIZE_CASES = ( + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Session.WorkingDirectory}}", + "steps": [STEP_TEMPLATE], + }, + 1, + id="session working directory not in 'name' scope", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_PATH], + "steps": [STEP_TEMPLATE], + }, + 1, + id="path parameter not in 'name' scope", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [STEP_TEMPLATE_FOO], + }, + 4, + id="step missing parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parmDef": [FOO_PARAMETER_INT], + "steps": [STEP_TEMPLATE_FOO], + }, + 5, # extra field + 2 param refs in each of command+args + id="key error and path parameter 'Foo' missing", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [STEP_TEMPLATE], + "jobEnvironments": [ENVIRONMENT_FOO], + }, + 2, + id="environment missing parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [STEP_TEMPLATE_FOO], + "jobEnvironments": [ENVIRONMENT_FOO], + }, + 6, + id="step and environment missing parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_PATH], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [STEP_TEMPLATE_FOO], + "name": "BarStep", + "script": make_script("Task", "BarScript", "echo"), + "parameterSpace": { + "taskParameterDefinitions": [ + { + "name": "Foo", + "type": "STRING", + "range": ["blah", "{{Param.Foo}}"], + } + ] + }, }, - 4, - id="step missing parameter", - ), - pytest.param( + ], + }, + 1, + id="parameter space cannot ref path parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ + STEP_TEMPLATE_FOO, { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parmDef": [FOO_PARAMETER_INT], - "steps": [STEP_TEMPLATE_FOO], + "name": "BarStep", + "script": make_script("Task", "BarScript", "echo {{Task.File.BazScript}}"), }, - 5, # extra field + 2 param refs in each of command+args - id="key error and path parameter 'Foo' missing", - ), - pytest.param( { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [STEP_TEMPLATE], - "jobEnvironments": [ENVIRONMENT_FOO], + "name": "BazStep", + "script": make_script("Task", "BazScript", "echo {{Task.File.BarScript}}"), }, - 2, - id="environment missing parameter", - ), - pytest.param( + ], + }, + 2, + id="multiple steps can't reference other files", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [STEP_TEMPLATE_FOO], + "jobEnvironments": [ + ENVIRONMENT_FOO, { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [STEP_TEMPLATE_FOO], - "jobEnvironments": [ENVIRONMENT_FOO], + "name": "BarEnv", + "script": make_script("Env", "BarScript", "echo {{Env.File.BazScript}}"), }, - 6, - id="step and environment missing parameter", - ), - pytest.param( { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_PATH], - "steps": [ - { - "name": "BarStep", - "script": make_script("Task", "BarScript", "echo"), - "parameterSpace": { - "taskParameterDefinitions": [ - { - "name": "Foo", - "type": "STRING", - "range": ["blah", "{{Param.Foo}}"], - } - ] - }, - }, - ], + "name": "BazEnv", + "script": make_script("Env", "BazScript", "echo {{Env.File.BarScript}}"), }, - 1, - id="parameter space cannot ref path parameter", - ), - pytest.param( + ], + }, + 2, + id="multiple environments can't reference other files", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ - STEP_TEMPLATE_FOO, + "name": "BarStep", + "script": make_script( + "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]} + ] + }, + "stepEnvironments": [ { - "name": "BarStep", - "script": make_script( - "Task", "BarScript", "echo {{Task.File.BazScript}}" - ), - }, - { - "name": "BazStep", + "name": "BarEnv", "script": make_script( - "Task", "BazScript", "echo {{Task.File.BarScript}}" + "Env", + "BarScript", + "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}} {{Task.Param.Foo}}", ), - }, + } ], }, - 2, - id="multiple steps can't reference other files", - ), - pytest.param( + ], + }, + 1, + id="step environment cannot see task params", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [STEP_TEMPLATE_FOO], - "jobEnvironments": [ - ENVIRONMENT_FOO, + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo {{Param.Foo}} {{Task.Param.Foo}} {{Env.File.BarScript}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]} + ] + }, + "stepEnvironments": [ { "name": "BarEnv", "script": make_script( - "Env", "BarScript", "echo {{Env.File.BazScript}}" - ), - }, - { - "name": "BazEnv", - "script": make_script( - "Env", "BazScript", "echo {{Env.File.BarScript}}" + "Env", + "BarScript", + "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", ), - }, + } ], }, - 2, - id="multiple environments can't reference other files", - ), - pytest.param( + ], + }, + 1, + id="step script cannot see environment", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" - ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]} - ] + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]}, + {"name": "Bar", "type": "FLOAT", "range": [1, 2]}, + { + "name": "Baz", + "type": "STRING", + "range": [ + "{{Task.Param.Foo}}", # Not other task params + "{{Task.Param.Bar}}", + "{{Task.Param.Baz}}", + "{{Env.File.BarScript}}", # Not env scripts + "{{Session.WorkingDirectory}}", # Not the session + ], }, - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}} {{Task.Param.Foo}}", - ), - } - ], - }, - ], - }, - 1, - id="step environment cannot see task params", - ), - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ + ] + }, + "stepEnvironments": [ { - "name": "BarStep", + "name": "BarEnv", "script": make_script( - "Task", + "Env", "BarScript", - "echo {{Param.Foo}} {{Task.Param.Foo}} {{Env.File.BarScript}}", + "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]} - ] - }, - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, + } ], }, - 1, - id="step script cannot see environment", - ), - pytest.param( + ], + }, + 5, + id="task params cannot see each other, envs, or the session", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": [1, 2]}, + {"name": "Bar", "type": "FLOAT", "range": [1, 2]}, + { + "name": "Baz", + "type": "STRING", + "range": [ + "{{Param.Foo}}", + "BazValue", + ], + }, + ] + }, + "stepEnvironments": [ { - "name": "BarStep", + "name": "BarEnv", "script": make_script( - "Task", + "Env", "BarScript", - "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}}", + "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]}, - {"name": "Bar", "type": "FLOAT", "range": [1, 2]}, - { - "name": "Baz", - "type": "STRING", - "range": [ - "{{Task.Param.Foo}}", # Not other task params - "{{Task.Param.Bar}}", - "{{Task.Param.Baz}}", - "{{Env.File.BarScript}}", # Not env scripts - "{{Session.WorkingDirectory}}", # Not the session - ], - }, - ] - }, - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, + } ], }, - 5, - id="task params cannot see each other, envs, or the session", - ), - pytest.param( { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", - "BarScript", - "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}}", - ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": [1, 2]}, - {"name": "Bar", "type": "FLOAT", "range": [1, 2]}, - { - "name": "Baz", - "type": "STRING", - "range": [ - "{{Param.Foo}}", - "BazValue", - ], - }, - ] + "name": "PubStep", + "script": make_script( + "Task", + "PubScript", + # errors: Task.Param.Foo, Task.Param.Bar, Task.Param.Baz + "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}} {{Task.Param.Pub1}} {{Task.Param.Pub2}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Pub1", "type": "INT", "range": [1, 2]}, + {"name": "Pub2", "type": "FLOAT", "range": [1, 2]}, + { + "name": "Pub3", + "type": "STRING", + "range": [ + # errors: Task.Param.Foo, Task.Param.Pub1 + "{{Task.Param.Foo}}", + "{{Task.Param.Pub1}}", + ], }, - "stepEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", - "BarScript", - "echo {{Param.Foo}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, + ] + }, + "stepEnvironments": [ { - "name": "PubStep", + "name": "PubEnv", "script": make_script( - "Task", + "Env", "PubScript", - # errors: Task.Param.Foo, Task.Param.Bar, Task.Param.Baz - "echo {{Param.Foo}} {{Task.Param.Foo}} {{Task.Param.Bar}} {{Task.Param.Baz}} {{Task.Param.Pub1}} {{Task.Param.Pub2}}", + # errors: Env.File.BarScript + "echo {{Param.Foo}} {{Env.File.PubScript}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Pub1", "type": "INT", "range": [1, 2]}, - {"name": "Pub2", "type": "FLOAT", "range": [1, 2]}, - { - "name": "Pub3", - "type": "STRING", - "range": [ - # errors: Task.Param.Foo, Task.Param.Pub1 - "{{Task.Param.Foo}}", - "{{Task.Param.Pub1}}", - ], - }, - ] - }, - "stepEnvironments": [ - { - "name": "PubEnv", - "script": make_script( - "Env", - "PubScript", - # errors: Env.File.BarScript - "echo {{Param.Foo}} {{Env.File.PubScript}} {{Env.File.BarScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, + } ], }, - 6, - id="task params cannot be seen across steps", - ), - pytest.param( + ], + }, + 6, + id="task params cannot be seen across steps", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_INT], + "jobEnvironments": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_INT], - "jobEnvironments": [ - { - "name": "BarEnv", - "script": make_script( - "Env", "BarScript", "echo {{Env.File.BarScript}}" - ), - }, - ], - "steps": [ - { - "name": "PubStep", - "script": make_script( - "Task", - "PubScript", - # errors: Env.File.BarScript, Env.File.PubScript - "echo {{Param.Foo}} {{Task.Param.Pub1}} {{Task.Param.Pub2}} {{Task.File.PubScript}}, {{Env.File.BarScript}} {{Env.File.PubScript}}", - ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Pub1", "type": "INT", "range": [1, 2]}, - {"name": "Pub2", "type": "FLOAT", "range": [1, 2]}, - ] - }, - "stepEnvironments": [ - { - "name": "PubEnv", - "script": make_script( - "Env", - "PubScript", - # errors: Env.File.BarScript, Task.File.PubScript - "echo {{Param.Foo}} {{Env.File.PubScript}} {{Env.File.BarScript}} {{Task.File.PubScript}} {{Session.WorkingDirectory}}", - ), - } - ], - }, - ], + "name": "BarEnv", + "script": make_script("Env", "BarScript", "echo {{Env.File.BarScript}}"), }, - 4, - id="files from job env cannot be seen in step scripts or step envs", - ), - pytest.param( + ], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [ + "name": "PubStep", + "script": make_script( + "Task", + "PubScript", + # errors: Env.File.BarScript, Env.File.PubScript + "echo {{Param.Foo}} {{Task.Param.Pub1}} {{Task.Param.Pub2}} {{Task.File.PubScript}}, {{Env.File.BarScript}} {{Env.File.PubScript}}", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Pub1", "type": "INT", "range": [1, 2]}, + {"name": "Pub2", "type": "FLOAT", "range": [1, 2]}, + ] + }, + "stepEnvironments": [ { - "name": "BarStep", + "name": "PubEnv", "script": make_script( - "Task", - "BarScript", - "echo 'Hi there''", + "Env", + "PubScript", + # errors: Env.File.BarScript, Task.File.PubScript + "echo {{Param.Foo}} {{Env.File.PubScript}} {{Env.File.BarScript}} {{Task.File.PubScript}} {{Session.WorkingDirectory}}", ), - "parameterSpace": { - "taskParameterDefinitions": [ - { - "name": "Baz", - "type": "STRING", - "range": [ - "{{Session.WorkingDirectory}}", # Not the session dir - "{{Session.HasPathMappingRules}}", # Not the path mapping reference - "{{Session.PathMappingRulesFile}}", # Not the path mapping file reference - ], - }, - ] - }, - }, + } ], }, - 3, - id="Cannot reference Session values in a Task Parameter", - ), - pytest.param( + ], + }, + 4, + id="files from job env cannot be seen in step scripts or step envs", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [ - { - "name": "BarStep", - "script": make_script( - "Task", - "BarScript", - "echo 'Hi there''", - ), - "hostRequirements": { - "amounts": [ - { - "name": "amount.{{Session.PathMappingRulesFile}}", - "min": 3, - }, - ], - "attributes": [ - { - "name": "attr.{{Session.WorkingDirectory}}", - "anyOf": ["{{Session.HasPathMappingRules}}"], - }, + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo 'Hi there''", + ), + "parameterSpace": { + "taskParameterDefinitions": [ + { + "name": "Baz", + "type": "STRING", + "range": [ + "{{Session.WorkingDirectory}}", # Not the session dir + "{{Session.HasPathMappingRules}}", # Not the path mapping reference + "{{Session.PathMappingRulesFile}}", # Not the path mapping file reference ], }, - }, - ], + ] + }, }, - 3, - id="Cannot reference Session values in a Capability", - ), - pytest.param( + ], + }, + 3, + id="Cannot reference Session values in a Task Parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "steps": [ - { - "name": "BarStep", - "stepEnvironments": [ - { - "name": "StepEnvA", - "script": { - "embeddedFiles": [ - { - "name": "fileA", - "type": "TEXT", - # References the other step env - "data": "{{Env.File.file1}}", - }, - ], - "actions": { - # References the other step env - "onEnter": {"command": "{{Env.File.file1}}"}, - }, - }, - }, - { - "name": "StepEnv1", - "script": { - "embeddedFiles": [ - { - "name": "file1", - "type": "TEXT", - # References the other step env - "data": "{{Env.File.fileA}}", - }, - ], - "actions": { - # References the other step env - "onEnter": {"command": "{{Env.File.fileA}}"}, - }, - }, - }, - ], - "script": { - "actions": {"onRun": {"command": "foo"}}, + "name": "BarStep", + "script": make_script( + "Task", + "BarScript", + "echo 'Hi there''", + ), + "hostRequirements": { + "amounts": [ + { + "name": "amount.{{Session.PathMappingRulesFile}}", + "min": 3, }, - }, - ], + ], + "attributes": [ + { + "name": "attr.{{Session.WorkingDirectory}}", + "anyOf": ["{{Session.HasPathMappingRules}}"], + }, + ], + }, }, - 4, - id="embedded env script files cannot reference other envs", - ), - pytest.param( + ], + }, + 3, + id="Cannot reference Session values in a Capability", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "jobEnvironments": [ + "name": "BarStep", + "stepEnvironments": [ { - "name": "BarEnv", - "script": make_script( - "Env", "BarScript", "echo {{Env.File.BarScript}}" - ), - }, - ], - "steps": [ - { - "name": "BarStep", - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Pub1", "type": "INT", "range": [1, 2]}, - {"name": "Pub2", "type": "FLOAT", "range": [1, 2]}, - ] - }, - "stepEnvironments": [ - { - "name": "StepEnvA", - "script": { - "embeddedFiles": [ - { - "name": "fileA", - "type": "TEXT", - "data": "filedata", - }, - ], - "actions": { - "onEnter": {"command": "filedata"}, - }, - }, - }, - { - "name": "StepEnv1", - "script": { - "actions": { - "onEnter": {"command": "command"}, - }, - }, - }, - ], + "name": "StepEnvA", "script": { - "actions": {"onRun": {"command": "foo"}}, "embeddedFiles": [ { - "name": "file1", + "name": "fileA", "type": "TEXT", - "data": "filedata", + # References the other step env + "data": "{{Env.File.file1}}", }, ], + "actions": { + # References the other step env + "onEnter": {"command": "{{Env.File.file1}}"}, + }, }, - "hostRequirements": { - "amounts": [ - { - # Reference a job environment file - "name": "amount.{{Env.File.BarScript}}", - "min": 3, - }, - ], - "attributes": [ + }, + { + "name": "StepEnv1", + "script": { + "embeddedFiles": [ { - # Reference a step script file - "name": "attr.{{Env.File.file1}}", - # Reference a step environment file, and two task parameters - "anyOf": [ - "{{Env.File.fileA}}", - "{{Task.Param.Pub1}} {{Task.Param.Pub2}}", - ], + "name": "file1", + "type": "TEXT", + # References the other step env + "data": "{{Env.File.fileA}}", }, ], + "actions": { + # References the other step env + "onEnter": {"command": "{{Env.File.fileA}}"}, + }, }, }, ], + "script": { + "actions": {"onRun": {"command": "foo"}}, + }, }, - 5, - id="Capabilities cannot reference envs or steps", - ), - pytest.param( + ], + }, + 4, + id="embedded env script files cannot reference other envs", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "jobEnvironments": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo", - "parameterDefinitions": [FOO_PARAMETER_STRING], - "steps": [STEP_TEMPLATE_FOO], - "jobEnvironments": [ - { - "name": "VariableEnv", - "variables": { - "VAR_NAME": "{{ Param.DoNotCreateAParamWithThisName }}", - }, - } - ], + "name": "BarEnv", + "script": make_script("Env", "BarScript", "echo {{Env.File.BarScript}}"), }, - 1, - id="Environment variable value referencing non-existant parameter", - ), - pytest.param( + ], + "steps": [ { - "specificationVersion": "jobtemplate-2023-09", - "name": "Foo {{Param.Foo}}", - "parameterDefinitions": [FOO_PARAMETER_INT], - "steps": [ + "name": "BarStep", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Pub1", "type": "INT", "range": [1, 2]}, + {"name": "Pub2", "type": "FLOAT", "range": [1, 2]}, + ] + }, + "stepEnvironments": [ { - "name": "BarStep", - "script": make_script( - "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" - ), - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": "1-{{Fake.Parameter}}"} - ] + "name": "StepEnvA", + "script": { + "embeddedFiles": [ + { + "name": "fileA", + "type": "TEXT", + "data": "filedata", + }, + ], + "actions": { + "onEnter": {"command": "filedata"}, + }, }, }, - ], - }, - 1, - id="int range expression fails on fake format string", - ), - # Test that we still properly collect parameter definitions for format string - # validation when we have a validation error in a parameter definition. - pytest.param( - { - "specificationVersion": "jobtemplate-2023-09", - "name": "DemoJob", - "parameterDefinitions": [ - {"name": "Foo", "type": "INT", "default": "Blah"}, - {"name": "Fuzz", "type": "INT"}, - ], - "steps": [ { - "name": "DemoStep", - "parameterSpace": { - "taskParameterDefinitions": [ - {"name": "Foo", "type": "INT", "range": "a-b"}, - {"name": "Fuzz", "type": "INT", "range": "1-10"}, - ] - }, + "name": "StepEnv1", "script": { "actions": { - "onRun": { - "command": "echo", - "args": [ - "{{Param.Foo}}", - "{{Param.Fuzz}}", - "{{Task.Param.Foo}}", - "{{Task.Param.Fuzz}}", - ], - } - } + "onEnter": {"command": "command"}, + }, }, - } + }, ], + "script": { + "actions": {"onRun": {"command": "foo"}}, + "embeddedFiles": [ + { + "name": "file1", + "type": "TEXT", + "data": "filedata", + }, + ], + }, + "hostRequirements": { + "amounts": [ + { + # Reference a job environment file + "name": "amount.{{Env.File.BarScript}}", + "min": 3, + }, + ], + "attributes": [ + { + # Reference a step script file + "name": "attr.{{Env.File.file1}}", + # Reference a step environment file, and two task parameters + "anyOf": [ + "{{Env.File.fileA}}", + "{{Task.Param.Pub1}} {{Task.Param.Pub2}}", + ], + }, + ], + }, }, - 2, # Validation of Job Foo & Task Foo - id="all parameter symbols are defined when validation errors", - ), - ), - ) - def test_parse_fails(self, data: dict[str, Any], error_count: int) -> None: - # Failure case testing for Open Job Description JobTemplate. + ], + }, + 5, + id="Capabilities cannot reference envs or steps", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo", + "parameterDefinitions": [FOO_PARAMETER_STRING], + "steps": [STEP_TEMPLATE_FOO], + "jobEnvironments": [ + { + "name": "VariableEnv", + "variables": { + "VAR_NAME": "{{ Param.DoNotCreateAParamWithThisName }}", + }, + } + ], + }, + 1, + id="Environment variable value referencing non-existant parameter", + ), + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "Foo {{Param.Foo}}", + "parameterDefinitions": [FOO_PARAMETER_INT], + "steps": [ + { + "name": "BarStep", + "script": make_script( + "Task", "BarScript", "echo {{Param.Foo}} {{Task.Param.Foo}}" + ), + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": "1-{{Fake.Parameter}}"} + ] + }, + }, + ], + }, + 1, + id="int range expression fails on fake format string", + ), + # Test that we still properly collect parameter definitions for format string + # validation when we have a validation error in a parameter definition. + pytest.param( + { + "specificationVersion": "jobtemplate-2023-09", + "name": "DemoJob", + "parameterDefinitions": [ + {"name": "Foo", "type": "INT", "default": "Blah"}, + {"name": "Fuzz", "type": "INT"}, + ], + "steps": [ + { + "name": "DemoStep", + "parameterSpace": { + "taskParameterDefinitions": [ + {"name": "Foo", "type": "INT", "range": "a-b"}, + {"name": "Fuzz", "type": "INT", "range": "1-10"}, + ] + }, + "script": { + "actions": { + "onRun": { + "command": "echo", + "args": [ + "{{Param.Foo}}", + "{{Param.Fuzz}}", + "{{Task.Param.Foo}}", + "{{Task.Param.Fuzz}}", + ], + } + } + }, + } + ], + }, + 2, # Validation of Job Foo & Task Foo + id="all parameter symbols are defined when validation errors", + ), +) + + +@pytest.mark.parametrize( + "data,error_count", + PARAMETRIZE_CASES, +) +def test_job_template_variables_parse_fails(data: dict[str, Any], error_count: int) -> None: + # Failure case testing for Open Job Description JobTemplate. - # WHEN - with pytest.raises(ValidationError) as excinfo: - _parse_model(model=JobTemplate, obj=data) + # WHEN + with pytest.raises(ValidationError) as excinfo: + _parse_model(model=JobTemplate, obj=data, context=ModelParsingContext_v2023_09()) - # THEN - assert len(excinfo.value.errors()) == error_count, str(excinfo.value) + # THEN + assert len(excinfo.value.errors()) == error_count, str(excinfo.value)