From a5117f3c5f5de6787aebea1ff440212d734e2402 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Fri, 20 Dec 2024 20:54:08 -0500 Subject: [PATCH 1/2] refactored configuration of perspectives --- configure.py | 40 +++++++++++-------- open-tofu/main.tf.template | 4 +- src/aws_lambda_mpic/__about__.py | 2 +- .../mpic_coordinator_lambda_function.py | 31 ++++++++------ tests/integration/test_deployed_mpic_api.py | 1 + .../test_mpic_coordinator_lambda.py | 28 +++++++++---- 6 files changed, 66 insertions(+), 40 deletions(-) diff --git a/configure.py b/configure.py index f852a64..8d7eca4 100755 --- a/configure.py +++ b/configure.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse +import json import os +from typing import Dict + import yaml import secrets import string @@ -34,7 +37,7 @@ def main(raw_args=None): # If the deployment id file does not exist, make a new one. if not os.path.isfile(args.deployment_id_file): with open(args.deployment_id_file, 'w') as stream: - deployment_id_to_write = ''.join(secrets.choice(string.digits) for i in range(10)) + deployment_id_to_write = ''.join(secrets.choice(string.digits) for _ in range(10)) stream.write(deployment_id_to_write) # Read the deployment id. @@ -63,29 +66,32 @@ def main(raw_args=None): if file.endswith(".generated.tf"): os.remove(os.path.join(open_tofu_dir, file)) - regions = config['perspectives'] - # Generate "main.generated.tf" based on main.tf.template. with open(args.main_tf_template) as stream: + perspective_config: Dict[str, dict] = {} + region_codes = config['perspectives'] + # Read the template file to a string. main_tf_string = stream.read() # Replace all the template vars used in the file. main_tf_string = main_tf_string.replace("{{api-region}}", config['api-region']) main_tf_string = main_tf_string.replace("{{deployment-id}}", str(deployment_id)) - - # Generate the region name list. - perspective_names_list = "|".join(config['perspectives']) - # Note the substitution uses quotes around the var. - main_tf_string = main_tf_string.replace("{{perspective-names-list}}", f"\"{perspective_names_list}\"") - - # Generate the ARNs list for validators. Note that this is not a list of actual ARN values. It is just a list of ARN names that will be substituted by Open Tofu. - arn_mpic_dcv_checker_list = "|".join([f"${{aws_lambda_function.mpic_dcv_checker_lambda_{region}.arn}}" for region in regions]) - main_tf_string = main_tf_string.replace("{{validator-arns-list}}", f"\"{arn_mpic_dcv_checker_list}\"") - - # Generate the ARNs list for CAA resolvers. Note that this is not a list of actual ARN values. It is just a list of ARN names that will be substituted by Open Tofu. - arn_mpic_caa_checker_list = "|".join([f"${{aws_lambda_function.mpic_caa_checker_lambda_{region}.arn}}" for region in regions]) - main_tf_string = main_tf_string.replace("{{mpic-caa-checker-arns-list}}", f"\"{arn_mpic_caa_checker_list}\"") + + # Construct the perspective configuration. + for region_code in region_codes: + perspective_endpoints = { + 'caa_endpoint_info': { + 'arn': f"${{aws_lambda_function.mpic_caa_checker_lambda_{region_code}.arn}}" + }, + 'dcv_endpoint_info': { + 'arn': f"${{aws_lambda_function.mpic_dcv_checker_lambda_{region_code}.arn}}" + } + } + perspective_config[region_code] = perspective_endpoints + + perspective_config_as_json = json.dumps(perspective_config, ensure_ascii=False) + main_tf_string = main_tf_string.replace("{{perspectives}}", perspective_config_as_json) # Replace default perspective count. main_tf_string = main_tf_string.replace("{{default-perspective-count}}", f"\"{config['default-perspective-count']}\"") @@ -97,7 +103,7 @@ def main(raw_args=None): main_tf_string = main_tf_string.replace("{{absolute-max-attempts-with-key}}", "") # Store the secret key for the vantage points hash in an environment variable. - hash_secret = ''.join(secrets.choice(string.ascii_letters) for i in range(20)) + hash_secret = ''.join(secrets.choice(string.ascii_letters) for _ in range(20)) main_tf_string = main_tf_string.replace("{{hash-secret}}", f"\"{hash_secret}\"") # Set the source path for the lambda functions. diff --git a/open-tofu/main.tf.template b/open-tofu/main.tf.template index ae7dbc2..6bcc387 100644 --- a/open-tofu/main.tf.template +++ b/open-tofu/main.tf.template @@ -80,9 +80,7 @@ resource "aws_lambda_function" "mpic_coordinator_lambda" { ] environment { variables = { - perspective_names = {{perspective-names-list}} - dcv_arns = {{validator-arns-list}} - caa_arns = {{mpic-caa-checker-arns-list}} + perspectives = jsonencode({{perspectives}}) default_perspective_count = {{default-perspective-count}} hash_secret = {{hash-secret}} {{absolute-max-attempts-with-key}} diff --git a/src/aws_lambda_mpic/__about__.py b/src/aws_lambda_mpic/__about__.py index 493f741..260c070 100644 --- a/src/aws_lambda_mpic/__about__.py +++ b/src/aws_lambda_mpic/__about__.py @@ -1 +1 @@ -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py index 49a2004..892a644 100644 --- a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py @@ -2,7 +2,7 @@ import yaml from aws_lambda_powertools.utilities.parser import event_parser, envelopes -from pydantic import TypeAdapter, ValidationError +from pydantic import TypeAdapter, ValidationError, BaseModel from open_mpic_core.common_domain.check_request import BaseCheckRequest from open_mpic_core.common_domain.check_response import CheckResponse from open_mpic_core.mpic_coordinator.domain.mpic_request import MpicRequest @@ -17,25 +17,32 @@ import json +class PerspectiveEndpointInfo(BaseModel): + arn: str + + +class PerspectiveEndpoints(BaseModel): + dcv_endpoint_info: PerspectiveEndpointInfo + caa_endpoint_info: PerspectiveEndpointInfo + + class MpicCoordinatorLambdaHandler: def __init__(self): - # load environment variables - self.all_target_perspectives = os.environ['perspective_names'].split("|") - self.dcv_arn_list = os.environ['dcv_arns'].split("|") - self.caa_arn_list = os.environ['caa_arns'].split("|") + perspectives_json = os.environ['perspectives'] + perspectives = {code: PerspectiveEndpoints.model_validate(endpoints) for code, endpoints in json.loads(perspectives_json).items()} + self.all_target_perspective_codes = list(perspectives.keys()) self.default_perspective_count = int(os.environ['default_perspective_count']) self.global_max_attempts = int(os.environ['absolute_max_attempts']) if 'absolute_max_attempts' in os.environ else None self.hash_secret = os.environ['hash_secret'] - self.arns_per_perspective_per_check_type = { - CheckType.DCV: {self.all_target_perspectives[i]: self.dcv_arn_list[i] for i in range(len(self.all_target_perspectives))}, - CheckType.CAA: {self.all_target_perspectives[i]: self.caa_arn_list[i] for i in range(len(self.all_target_perspectives))} + self.remotes_per_perspective_per_check_type = { + CheckType.DCV: {perspective_code: perspective_config.dcv_endpoint_info for perspective_code, perspective_config in perspectives.items()}, + CheckType.CAA: {perspective_code: perspective_config.caa_endpoint_info for perspective_code, perspective_config in perspectives.items()} } - all_target_perspective_codes = self.all_target_perspectives all_possible_perspectives_by_code = MpicCoordinatorLambdaHandler.load_aws_region_config() self.target_perspectives = MpicCoordinatorLambdaHandler.convert_codes_to_remote_perspectives( - all_target_perspective_codes, all_possible_perspectives_by_code) + self.all_target_perspective_codes, all_possible_perspectives_by_code) self.mpic_coordinator_configuration = MpicCoordinatorConfiguration( self.target_perspectives, @@ -84,9 +91,9 @@ def convert_codes_to_remote_perspectives(perspective_codes: list[str], def call_remote_perspective(self, perspective: RemotePerspective, check_type: CheckType, check_request: BaseCheckRequest) -> CheckResponse: # Uses dcv_arn_list, caa_arn_list client = boto3.client('lambda', perspective.code) - function_name = self.arns_per_perspective_per_check_type[check_type][perspective.code] + function_endpoint_info = self.remotes_per_perspective_per_check_type[check_type][perspective.code] response = client.invoke( # AWS Lambda-specific structure - FunctionName=function_name, + FunctionName=function_endpoint_info.arn, InvocationType='RequestResponse', Payload=check_request.model_dump_json() # AWS Lambda functions expect a JSON string for payload ) diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index e4e6999..42777cf 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -48,6 +48,7 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se # assert response body has a list of perspectives with length 2, and each element has response code 200 mpic_response = self.mpic_response_adapter.validate_json(response.text) print("\nResponse:\n", json.dumps(mpic_response.model_dump(), indent=4)) # pretty print response body + assert mpic_response.is_valid is True perspectives_list = mpic_response.perspectives assert len(perspectives_list) == request.orchestration_parameters.perspective_count assert (len(list(filter(lambda perspective: perspective.check_type == CheckType.CAA, perspectives_list))) diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index 4fa0e32..4ae6a1c 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -21,6 +21,7 @@ from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import MpicCoordinatorLambdaHandler from botocore.response import StreamingBody import aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function as mpic_coordinator_lambda_function +from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import PerspectiveEndpoints, PerspectiveEndpointInfo from unit.test_util.valid_check_creator import ValidCheckCreator from unit.test_util.valid_mpic_request_creator import ValidMpicRequestCreator @@ -31,10 +32,22 @@ class TestMpicCoordinatorLambda: @staticmethod @pytest.fixture(scope='class') def set_env_variables(): + perspectives_as_dict = { + "us-east-1": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-east-1:123456789012:caa/us-east-1'), + dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-east-1:123456789012:dcv/us-east-1')), + "us-west-1": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-west-1:123456789012:caa/us-west-1'), + dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-west-1:123456789012:dcv/us-west-1')), + "eu-west-2": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-west-2:123456789012:caa/eu-west-2'), + dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-west-2:123456789012:dcv/eu-west-2')), + "eu-central-2": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-central-2:123456789012:caa/eu-central-2'), + dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-central-2:123456789012:dcv/eu-central-2')), + "ap-northeast-1": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-northeast-1:123456789012:caa/ap-northeast-1'), + dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-northeast-1:123456789012:dcv/ap-northeast-1')), + "ap-south-2": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-south-2:123456789012:caa/ap-south-2'), + dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-south-2:123456789012:dcv/ap-south-2')) + } envvars = { - 'perspective_names': 'us-east-1|us-west-1|eu-west-2|eu-central-2|ap-northeast-1|ap-south-2', - 'dcv_arns': 'arn:aws:acm-pca:us-east-1:123456789012:validator/us-east-1|arn:aws:acm-pca:us-west-1:123456789012:validator/us-west-1|arn:aws:acm-pca:eu-west-2:123456789012:validator/eu-west-2|arn:aws:acm-pca:eu-central-2:123456789012:validator/eu-central-2|arn:aws:acm-pca:ap-northeast-1:123456789012:validator/ap-northeast-1|arn:aws:acm-pca:ap-south-2:123456789012:validator/ap-south-2', - 'caa_arns': 'arn:aws:acm-pca:us-east-1:123456789012:caa/us-east-1|arn:aws:acm-pca:us-west-1:123456789012:caa/us-west-1|arn:aws:acm-pca:eu-west-2:123456789012:caa/eu-west-2|arn:aws:acm-pca:eu-central-2:123456789012:caa/eu-central-2|arn:aws:acm-pca:ap-northeast-1:123456789012:caa/ap-northeast-1|arn:aws:acm-pca:ap-south-2:123456789012:caa/ap-south-2', + 'perspectives': json.dumps({k: v.model_dump() for k, v in perspectives_as_dict.items()}), 'default_perspective_count': '3', 'hash_secret': 'test_secret' } @@ -56,6 +69,7 @@ def call_remote_perspective__should_make_aws_lambda_call_with_provided_arguments def lambda_handler__should_return_400_error_and_details_given_invalid_request_body(self): request = ValidMpicRequestCreator.create_valid_dcv_mpic_request() + # noinspection PyTypeChecker request.domain_or_ip_target = None api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = request.model_dump_json() @@ -118,11 +132,11 @@ def load_aws_region_config__should_return_dict_of_aws_regions_with_proximity_inf def constructor__should_initialize_mpic_coordinator_and_set_target_perspectives(self, set_env_variables): mpic_coordinator_lambda_handler = MpicCoordinatorLambdaHandler() all_possible_perspectives = TestMpicCoordinatorLambda.get_perspectives_by_code_dict_from_file() - named_perspectives = os.environ['perspective_names'].split('|') + for target_perspective in mpic_coordinator_lambda_handler.target_perspectives: + assert target_perspective in all_possible_perspectives.values() mpic_coordinator = mpic_coordinator_lambda_handler.mpic_coordinator - us_east_1_perspective = all_possible_perspectives['us-east-1'] - assert us_east_1_perspective in mpic_coordinator.target_perspectives - assert len(named_perspectives) == len(mpic_coordinator.target_perspectives) + assert len(mpic_coordinator.target_perspectives) == 6 + assert mpic_coordinator.default_perspective_count == 3 assert mpic_coordinator.hash_secret == 'test_secret' # noinspection PyUnusedLocal From e22557cca7b60b9644a6a0cda6c7c756e790b756 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Sat, 21 Dec 2024 15:45:57 -0500 Subject: [PATCH 2/2] addressed warnings in unit tests. also moved to main PyPI index instead of test --- pyproject.toml | 10 +++++----- tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py | 3 +-- .../aws_lambda_mpic/test_mpic_coordinator_lambda.py | 10 ++++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4658f13..7e293ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "dnspython==2.6.1", "pydantic==2.8.2", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==3.0.0", + "open-mpic-core==3.0.1", ] [project.optional-dependencies] @@ -66,7 +66,7 @@ path="venv" [tool.hatch.envs.default.env-vars] PIP_INDEX_URL = "https://pypi.org/simple/" -PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" # FIXME here temporarily to test open-mpic-core packaging +#PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" # FIXME here temporarily to test open-mpic-core packaging PIP_VERBOSE = "1" [tool.hatch.envs.lambda-layer] @@ -76,7 +76,7 @@ type="virtual" path="layer/create_layer_virtualenv" [tool.hatch.envs.lambda-layer.env-vars] -PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" +#PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" PIP_ONLY_BINARY = ":all:" #PIP_PLATFORM = "manylinux2014_aarch64" #PIP_TARGET = "layer/create_layer_virtualenv2/lib/python3.11/site-packages" # does not work... bug in pip 24.2? @@ -92,8 +92,8 @@ features = [ ] installer = "pip" -[tool.hatch.envs.test.env-vars] -PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" +#[tool.hatch.envs.test.env-vars] +#PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" [tool.hatch.envs.test.scripts] pre-install = "python -m ensurepip" diff --git a/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py index 7751bf5..9a187e4 100644 --- a/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py @@ -38,8 +38,7 @@ def lambda_handler__should_do_caa_check_using_configured_caa_checker(self, set_e def create_caa_check_response(): return CaaCheckResponse(perspective_code='us-east-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=True, - found_at='example.com', - response='dummy_response'), + found_at='example.com'), timestamp_ns=time.time_ns()) diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index 4ae6a1c..0cc0711 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -1,6 +1,5 @@ import io import json -import os from datetime import datetime from importlib import resources @@ -73,6 +72,7 @@ def lambda_handler__should_return_400_error_and_details_given_invalid_request_bo request.domain_or_ip_target = None api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = request.model_dump_json() + # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result['statusCode'] == 400 result_body = json.loads(result['body']) @@ -83,18 +83,22 @@ def lambda_handler__should_return_400_error_and_details_given_invalid_check_type request.check_type = 'invalid_check_type' api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = request.model_dump_json() + # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result['statusCode'] == 400 result_body = json.loads(result['body']) - assert result_body['validation_issues'][0]['type'] == 'union_tag_invalid' + assert result_body['validation_issues'][0]['type'] == 'literal_error' def lambda_handler__should_return_400_error_given_logically_invalid_request(self): request = ValidMpicRequestCreator.create_valid_dcv_mpic_request() request.orchestration_parameters.perspective_count = 1 api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = request.model_dump_json() + # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result['statusCode'] == 400 + result_body = json.loads(result['body']) + assert result_body['validation_issues'][0]['issue_type'] == 'invalid-perspective-count' def lambda_handler__should_return_500_error_given_other_unexpected_errors(self, set_env_variables, mocker): request = ValidMpicRequestCreator.create_valid_dcv_mpic_request() @@ -102,6 +106,7 @@ def lambda_handler__should_return_500_error_given_other_unexpected_errors(self, api_request.body = request.model_dump_json() mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.coordinate_mpic', side_effect=Exception('Something went wrong')) + # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result['statusCode'] == 500 @@ -117,6 +122,7 @@ def lambda_handler__should_coordinate_mpic_using_configured_mpic_coordinator(sel 'headers': {'Content-Type': 'application/json'}, 'body': mock_return_value.model_dump_json() } + # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result == expected_response