From 026bc006db59605cfdd47fc935675d18d2dbcbc6 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 28 Jan 2025 16:05:44 -0500 Subject: [PATCH 01/10] ran Black formatter --- pyproject.toml | 6 +- .../mpic_caa_checker_lambda_function.py | 26 +- .../mpic_coordinator_lambda_function.py | 103 ++++---- .../mpic_dcv_checker_lambda_function.py | 23 +- tests/integration/test_deployed_mpic_api.py | 140 +++++++---- tests/integration/testing_api_client.py | 13 +- tests/unit/aws_lambda_mpic/conftest.py | 2 +- .../test_caa_checker_lambda.py | 41 ++- .../test_dcv_checker_lambda.py | 59 +++-- .../test_mpic_coordinator_lambda.py | 233 ++++++++++-------- 10 files changed, 371 insertions(+), 275 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae7f586..9a6887c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,8 +33,9 @@ dependencies = [ "pydantic==2.8.2", "aiohttp==3.11.11", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==4.6.1", + "open-mpic-core==4.6.2", "aioboto3~=13.3.0", + "black==24.8.0", ] [project.optional-dependencies] @@ -135,3 +136,6 @@ include_namespace_packages = true omit = [ "*/src/*/__about__.py", ] + +[tool.black] +line-length = 120 diff --git a/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py b/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py index 87df51d..777b9fa 100644 --- a/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py @@ -3,26 +3,28 @@ from aws_lambda_powertools.utilities.parser import event_parser -from open_mpic_core.common_domain.check_request import CaaCheckRequest -from open_mpic_core.mpic_caa_checker.mpic_caa_checker import MpicCaaChecker -from open_mpic_core.common_util.trace_level_logger import get_logger +from open_mpic_core import CaaCheckRequest +from open_mpic_core import MpicCaaChecker +from open_mpic_core import get_logger logger = get_logger(__name__) class MpicCaaCheckerLambdaHandler: def __init__(self): - self.perspective_code = os.environ['AWS_REGION'] - self.default_caa_domain_list = os.environ['default_caa_domains'].split("|") - self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None + self.perspective_code = os.environ["AWS_REGION"] + self.default_caa_domain_list = os.environ["default_caa_domains"].split("|") + self.log_level = os.environ["log_level"] if "log_level" in os.environ else None self.logger = logger.getChild(self.__class__.__name__) if self.log_level: self.logger.setLevel(self.log_level) - self.caa_checker = MpicCaaChecker(default_caa_domain_list=self.default_caa_domain_list, - perspective_code=self.perspective_code, - log_level=self.logger.level) + self.caa_checker = MpicCaaChecker( + default_caa_domain_list=self.default_caa_domain_list, + perspective_code=self.perspective_code, + log_level=self.logger.level, + ) def process_invocation(self, caa_request: CaaCheckRequest): try: @@ -34,9 +36,9 @@ def process_invocation(self, caa_request: CaaCheckRequest): caa_response = event_loop.run_until_complete(self.caa_checker.check_caa(caa_request)) result = { - 'statusCode': 200, # note: must be snakeCase - 'headers': {'Content-Type': 'application/json'}, - 'body': caa_response.model_dump_json() + "statusCode": 200, # note: must be snakeCase + "headers": {"Content-Type": "application/json"}, + "body": caa_response.model_dump_json(), } return result diff --git a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py index 18aeb6c..0184468 100644 --- a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py @@ -1,4 +1,3 @@ -import logging import os import json import traceback @@ -13,15 +12,12 @@ from pydantic import TypeAdapter, ValidationError, BaseModel from aws_lambda_powertools.utilities.parser import event_parser, envelopes -from open_mpic_core.common_domain.check_request import BaseCheckRequest -from open_mpic_core.common_domain.check_response import CheckResponse -from open_mpic_core.mpic_coordinator.domain.mpic_request import MpicRequest -from open_mpic_core.mpic_coordinator.domain.mpic_request_validation_error import MpicRequestValidationError -from open_mpic_core.mpic_coordinator.messages.mpic_request_validation_messages import MpicRequestValidationMessages -from open_mpic_core.mpic_coordinator.mpic_coordinator import MpicCoordinator, MpicCoordinatorConfiguration -from open_mpic_core.common_domain.enum.check_type import CheckType -from open_mpic_core.mpic_coordinator.domain.remote_perspective import RemotePerspective -from open_mpic_core.common_util.trace_level_logger import get_logger +from open_mpic_core import MpicRequest, CheckRequest, CheckResponse +from open_mpic_core import MpicRequestValidationError, MpicRequestValidationMessages +from open_mpic_core import MpicCoordinator, MpicCoordinatorConfiguration +from open_mpic_core import CheckType +from open_mpic_core import RemotePerspective +from open_mpic_core import get_logger logger = get_logger(__name__) @@ -37,35 +33,46 @@ class PerspectiveEndpoints(BaseModel): class MpicCoordinatorLambdaHandler: def __init__(self): - perspectives_json = os.environ['perspectives'] - perspectives = {code: PerspectiveEndpoints.model_validate(endpoints) for code, endpoints in json.loads(perspectives_json).items()} + perspectives_json = os.environ["perspectives"] + perspectives = { + code: PerspectiveEndpoints.model_validate(endpoints) + for code, endpoints in json.loads(perspectives_json).items() + } self._all_target_perspective_codes = list(perspectives.keys()) - self.default_perspective_count = int(os.environ['default_perspective_count']) - self.global_max_attempts = int(os.environ['absolute_max_attempts']) if 'absolute_max_attempts' in os.environ else None - self.hash_secret = os.environ['hash_secret'] - self.log_level = os.getenv('log_level', None) + self.default_perspective_count = int(os.environ["default_perspective_count"]) + self.global_max_attempts = ( + int(os.environ["absolute_max_attempts"]) if "absolute_max_attempts" in os.environ else None + ) + self.hash_secret = os.environ["hash_secret"] + self.log_level = os.getenv("log_level", None) self.logger = logger.getChild(self.__class__.__name__) if self.log_level: self.logger.setLevel(self.log_level) self.remotes_per_perspective_per_check_type = { - CheckType.DCV: {perspective_code: perspective_config.dcv_endpoint_info for perspective_code, perspective_config in perspectives.items()}, - CheckType.CAA: {perspective_code: perspective_config.caa_endpoint_info for perspective_code, perspective_config in perspectives.items()} + CheckType.DCV: { + perspective_code: perspective_config.dcv_endpoint_info + for perspective_code, perspective_config in perspectives.items() + }, + CheckType.CAA: { + perspective_code: perspective_config.caa_endpoint_info + for perspective_code, perspective_config in perspectives.items() + }, } all_possible_perspectives_by_code = MpicCoordinatorLambdaHandler.load_aws_region_config() self.target_perspectives = MpicCoordinatorLambdaHandler.convert_codes_to_remote_perspectives( - self._all_target_perspective_codes, all_possible_perspectives_by_code) + self._all_target_perspective_codes, all_possible_perspectives_by_code + ) self.mpic_coordinator_configuration = MpicCoordinatorConfiguration( - self.target_perspectives, - self.default_perspective_count, - self.global_max_attempts, - self.hash_secret + self.target_perspectives, self.default_perspective_count, self.global_max_attempts, self.hash_secret ) - self.mpic_coordinator = MpicCoordinator(self.call_remote_perspective, self.mpic_coordinator_configuration, self.logger.level) + self.mpic_coordinator = MpicCoordinator( + self.call_remote_perspective, self.mpic_coordinator_configuration, self.logger.level + ) # for correct deserialization of responses based on discriminator field (check type) self.mpic_request_adapter = TypeAdapter(MpicRequest) @@ -78,7 +85,7 @@ async def initialize_client_pools(self): # Call this during cold start for perspective_code in self._all_target_perspective_codes: for _ in range(10): # pre-populate pool - client = await self._session.client('lambda', perspective_code).__aenter__() + client = await self._session.client("lambda", perspective_code).__aenter__() await self._client_pools[perspective_code].put(client) async def get_lambda_client(self, perspective_code: str): @@ -99,16 +106,17 @@ def load_aws_region_config() -> dict[str, RemotePerspective]: Reads in the available perspectives from a configuration yaml and returns them as a dict (map). :return: dict of available perspectives with region code as key """ - with resources.files('resources').joinpath('aws_region_config.yaml').open('r') as file: + with resources.files("resources").joinpath("aws_region_config.yaml").open("r") as file: aws_region_config_yaml = yaml.safe_load(file) aws_region_type_adapter = TypeAdapter(list[RemotePerspective]) - aws_regions_list = aws_region_type_adapter.validate_python(aws_region_config_yaml['aws_available_regions']) + aws_regions_list = aws_region_type_adapter.validate_python(aws_region_config_yaml["aws_available_regions"]) aws_regions_dict = {region.code: region for region in aws_regions_list} return aws_regions_dict @staticmethod - def convert_codes_to_remote_perspectives(perspective_codes: list[str], - all_possible_perspectives_by_code: dict[str, RemotePerspective]) -> list[RemotePerspective]: + def convert_codes_to_remote_perspectives( + perspective_codes: list[str], all_possible_perspectives_by_code: dict[str, RemotePerspective] + ) -> list[RemotePerspective]: remote_perspectives = [] for perspective_code in perspective_codes: @@ -121,19 +129,21 @@ def convert_codes_to_remote_perspectives(perspective_codes: list[str], return remote_perspectives # This function MUST validate its response and return a proper open_mpic_core object type. - async def call_remote_perspective(self, perspective: RemotePerspective, check_type: CheckType, check_request: BaseCheckRequest) -> CheckResponse: + async def call_remote_perspective( + self, perspective: RemotePerspective, check_type: CheckType, check_request: CheckRequest + ) -> CheckResponse: client = await self.get_lambda_client(perspective.code) try: function_endpoint_info = self.remotes_per_perspective_per_check_type[check_type][perspective.code] response = await client.invoke( # AWS Lambda-specific structure FunctionName=function_endpoint_info.arn, - InvocationType='RequestResponse', - Payload=check_request.model_dump_json() # AWS Lambda functions expect a JSON string for payload + InvocationType="RequestResponse", + Payload=check_request.model_dump_json(), # AWS Lambda functions expect a JSON string for payload ) - response_payload = json.loads(await response['Payload'].read()) - return self.check_response_adapter.validate_json(response_payload['body']) + response_payload = json.loads(await response["Payload"].read()) + return self.check_response_adapter.validate_json(response_payload["body"]) except ValidationError as ve: - self.logger.log(level=logging.ERROR, msg=f"Validation error in response from {perspective.code}: {ve}") + self.logger.error(msg=f"Validation error in response from {perspective.code}: {ve}") raise ve finally: await self.release_lambda_client(perspective.code, client) @@ -141,9 +151,9 @@ async def call_remote_perspective(self, perspective: RemotePerspective, check_ty async def process_invocation(self, mpic_request: MpicRequest) -> dict: mpic_response = await self.mpic_coordinator.coordinate_mpic(mpic_request) return { - 'statusCode': 200, - 'headers': {'Content-Type': 'application/json'}, - 'body': mpic_response.model_dump_json() + "statusCode": 200, + "headers": {"Content-Type": "application/json"}, + "body": mpic_response.model_dump_json(), } @@ -178,9 +188,9 @@ def get_handler() -> MpicCoordinatorLambdaHandler: def handle_lambda_exceptions(func): def build_400_response(error_name, issues_list): return { - 'statusCode': 400, - 'headers': {'Content-Type': 'application/json'}, - 'body': json.dumps({'error': error_name, 'validation_issues': issues_list}) + "statusCode": 400, + "headers": {"Content-Type": "application/json"}, + "body": json.dumps({"error": error_name, "validation_issues": issues_list}), } def wrapper(*args, **kwargs): @@ -190,16 +200,19 @@ def wrapper(*args, **kwargs): validation_issues = json.loads(e.__notes__[0]) return build_400_response(MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key, validation_issues) except ValidationError as validation_error: - return build_400_response(MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key, validation_error.errors()) + return build_400_response( + MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key, validation_error.errors() + ) except Exception as e: logger.error(f"An error occurred: {str(e)}") print(traceback.format_exc()) print(f"BOY HOWDY error occurred: {str(e)}") return { - 'statusCode': 500, - 'headers': {'Content-Type': 'application/json'}, - 'body': json.dumps({'error': str(e)}) + "statusCode": 500, + "headers": {"Content-Type": "application/json"}, + "body": json.dumps({"error": str(e)}), } + return wrapper diff --git a/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py b/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py index ef6cf1f..64b931a 100644 --- a/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py @@ -3,25 +3,24 @@ from aws_lambda_powertools.utilities.parser import event_parser -from open_mpic_core.common_domain.check_request import DcvCheckRequest -from open_mpic_core.mpic_dcv_checker.mpic_dcv_checker import MpicDcvChecker -from open_mpic_core.common_util.trace_level_logger import get_logger +from open_mpic_core import DcvCheckRequest, MpicDcvChecker +from open_mpic_core import get_logger logger = get_logger(__name__) class MpicDcvCheckerLambdaHandler: def __init__(self): - self.perspective_code = os.environ['AWS_REGION'] - self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None + self.perspective_code = os.environ["AWS_REGION"] + self.log_level = os.environ["log_level"] if "log_level" in os.environ else None self.logger = logger.getChild(self.__class__.__name__) if self.log_level: self.logger.setLevel(self.log_level) - self.dcv_checker = MpicDcvChecker(perspective_code=self.perspective_code, - reuse_http_client=False, - log_level=self.logger.level) + self.dcv_checker = MpicDcvChecker( + perspective_code=self.perspective_code, reuse_http_client=False, log_level=self.logger.level + ) def process_invocation(self, dcv_request: DcvCheckRequest): try: @@ -37,14 +36,14 @@ def process_invocation(self, dcv_request: DcvCheckRequest): dcv_response = event_loop.run_until_complete(self.dcv_checker.check_dcv(dcv_request)) status_code = 200 if dcv_response.errors is not None and len(dcv_response.errors) > 0: - if dcv_response.errors[0].error_type == '404': + if dcv_response.errors[0].error_type == "404": status_code = 404 else: status_code = 500 result = { - 'statusCode': status_code, - 'headers': {'Content-Type': 'application/json'}, - 'body': dcv_response.model_dump_json() + "statusCode": status_code, + "headers": {"Content-Type": "application/json"}, + "body": dcv_response.model_dump_json(), } return result diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index a14ce96..c747c56 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -1,22 +1,23 @@ import json import sys import pytest -from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod from pydantic import TypeAdapter -from open_mpic_core.common_domain.check_parameters import CaaCheckParameters, DcvWebsiteChangeValidationDetails, DcvAcmeDns01ValidationDetails, DcvDnsChangeValidationDetails -from open_mpic_core.common_domain.check_parameters import DcvCheckParameters -from open_mpic_core.common_domain.enum.certificate_type import CertificateType -from open_mpic_core.common_domain.enum.check_type import CheckType -from open_mpic_core.common_domain.enum.dns_record_type import DnsRecordType -from open_mpic_core.mpic_coordinator.domain.mpic_request import MpicCaaRequest -from open_mpic_core.mpic_coordinator.domain.mpic_request import MpicDcvRequest -from open_mpic_core.mpic_coordinator.domain.mpic_orchestration_parameters import MpicRequestOrchestrationParameters +from open_mpic_core import ( + CaaCheckParameters, + DcvWebsiteChangeValidationDetails, + DcvAcmeDns01ValidationDetails, + DcvDnsChangeValidationDetails, +) +from open_mpic_core import DcvCheckParameters +from open_mpic_core import CertificateType, CheckType, DnsRecordType +from open_mpic_core import MpicCaaRequest, MpicDcvRequest, MpicResponse +from open_mpic_core import MpicRequestOrchestrationParameters +from open_mpic_core import MpicRequestValidationMessages import testing_api_client -from open_mpic_core.mpic_coordinator.domain.mpic_response import MpicResponse -from open_mpic_core.mpic_coordinator.messages.mpic_request_validation_messages import MpicRequestValidationMessages + MPIC_REQUEST_PATH = "/mpic" @@ -28,20 +29,22 @@ class TestDeployedMpicApi: def setup_class(cls): cls.mpic_response_adapter = TypeAdapter(MpicResponse) - @pytest.fixture(scope='class') + @pytest.fixture(scope="class") def api_client(self): with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: # blank out argv except first param; arg parser doesn't expect pytest args - class_scoped_monkeypatch.setattr(sys, 'argv', sys.argv[:1]) + class_scoped_monkeypatch.setattr(sys, "argv", sys.argv[:1]) api_client = testing_api_client.TestingApiClient() yield api_client api_client.close() def api_should_return_200_and_passed_corroboration_given_successful_caa_check(self, api_client): request = MpicCaaRequest( - domain_or_ip_target='example.com', + domain_or_ip_target="example.com", orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - caa_check_parameters=CaaCheckParameters(certificate_type=CertificateType.TLS_SERVER, caa_domains=['mozilla.com']) + caa_check_parameters=CaaCheckParameters( + certificate_type=CertificateType.TLS_SERVER, caa_domains=["mozilla.com"] + ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body @@ -54,9 +57,12 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se assert mpic_response.is_valid is True perspectives_list = mpic_response.perspectives assert len(perspectives_list) == request.orchestration_parameters.perspective_count - assert (len(list(filter(lambda perspective: perspective.check_type == CheckType.CAA, perspectives_list))) - == request.orchestration_parameters.perspective_count) + assert ( + len(list(filter(lambda perspective: perspective.check_type == CheckType.CAA, perspectives_list))) + == request.orchestration_parameters.perspective_count + ) + # fmt: off @pytest.mark.parametrize('domain_or_ip_target, purpose_of_test, is_wildcard_domain', [ ('empty.basic.caatestsuite.com', 'Tests handling of 0 issue ";"', False), ('deny.basic.caatestsuite.com', 'Tests handling of 0 issue "caatestsuite.com"', False), @@ -81,8 +87,10 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se ('refused.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning REFUSED', False), ('xss.caatestsuite.com', 'Tests rejection when issue property has HTML and JS', False), ]) - def api_should_return_is_valid_false_for_all_tests_in_do_not_issue_caa_test_suite(self, api_client, domain_or_ip_target, - purpose_of_test, is_wildcard_domain): + # fmt: on + def api_should_return_is_valid_false_for_all_tests_in_do_not_issue_caa_test_suite( + self, api_client, domain_or_ip_target, purpose_of_test, is_wildcard_domain + ): print(f"Running test for {domain_or_ip_target} ({purpose_of_test})") if is_wildcard_domain: domain_or_ip_target = "*." + domain_or_ip_target @@ -90,13 +98,14 @@ def api_should_return_is_valid_false_for_all_tests_in_do_not_issue_caa_test_suit domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), caa_check_parameters=CaaCheckParameters( - certificate_type=CertificateType.TLS_SERVER, caa_domains=['example.com'] - ) + certificate_type=CertificateType.TLS_SERVER, caa_domains=["example.com"] + ), ) response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) mpic_response = self.mpic_response_adapter.validate_json(response.text) assert mpic_response.is_valid is False + # fmt: off # NOTE: Open MPIC AWS-Lambda-Python currently is not able to communicate with an IPv6 only nameserver. # This case is handled in a compliant manner as it is treated as a lookup failure. # The test for proper communication with an IPv6 nameserver can be enabled with the following additional parameter to the list below. @@ -116,8 +125,10 @@ def api_should_return_is_valid_false_for_all_tests_in_do_not_issue_caa_test_suit ('permit.basic.caatestsuite.com', 'Tests acceptance when name contains a permissible CAA record set', False), ('deny.permit.basic.caatestsuite.com', 'Tests acceptance on a CAA record set', False), ]) - def api_should_return_is_valid_true_for_valid_tests_in_caa_test_suite_when_caa_domain_is_caatestsuite_com(self, api_client, domain_or_ip_target, - purpose_of_test, is_wildcard_domain): + # fmt: on + def api_should_return_is_valid_true_for_valid_tests_in_caa_test_suite_when_caa_domain_is_caatestsuite_com( + self, api_client, domain_or_ip_target, purpose_of_test, is_wildcard_domain + ): print(f"Running test for {domain_or_ip_target} ({purpose_of_test})") if is_wildcard_domain: domain_or_ip_target = "*." + domain_or_ip_target @@ -125,85 +136,108 @@ def api_should_return_is_valid_true_for_valid_tests_in_caa_test_suite_when_caa_d domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), caa_check_parameters=CaaCheckParameters( - certificate_type=CertificateType.TLS_SERVER, caa_domains=['caatestsuite.com', 'example.com']) + certificate_type=CertificateType.TLS_SERVER, caa_domains=["caatestsuite.com", "example.com"] + ), ) response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) mpic_response = self.mpic_response_adapter.validate_json(response.text) assert mpic_response.is_valid is True + # fmt: off @pytest.mark.skip(reason='Behavior not required in RFC 8659') @pytest.mark.parametrize('domain_or_ip_target, purpose_of_test', [ ('dname-deny.basic.caatestsuite.com', 'Tests handling of a DNAME when CAA record exists at DNAME target'), ('cname-deny-sub.basic.caatestsuite.com', 'Tests handling of a CNAME when CAA record exists at parent of CNAME target') ]) - def api_should_return_is_valid_false_for_do_not_issue_caa_test_suite_for_rfc_6844(self, api_client, domain_or_ip_target, purpose_of_test): + # fmt: on + def api_should_return_is_valid_false_for_do_not_issue_caa_test_suite_for_rfc_6844( + self, api_client, domain_or_ip_target, purpose_of_test + ): print(f"Running test for {domain_or_ip_target} ({purpose_of_test})") request = MpicCaaRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - caa_check_parameters=CaaCheckParameters(certificate_type=CertificateType.TLS_SERVER, caa_domains=['example.com']) + caa_check_parameters=CaaCheckParameters( + certificate_type=CertificateType.TLS_SERVER, caa_domains=["example.com"] + ), ) response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) mpic_response = self.mpic_response_adapter.validate_json(response.text) assert mpic_response.is_valid is False + # fmt: off @pytest.mark.parametrize('domain_or_ip_target, purpose_of_test', [ ('dns-01.integration-testing.open-mpic.org', 'Standard proper dns-01 test'), ('dns-01-multi.integration-testing.open-mpic.org', 'Proper dns-01 test with multiple TXT records'), ('dns-01-cname.integration-testing.open-mpic.org', 'Proper dns-01 test with CNAME') ]) + # fmt: on def api_should_return_200_given_valid_dns_01_validation(self, api_client, domain_or_ip_target, purpose_of_test): print(f"Running test for {domain_or_ip_target} ({purpose_of_test})") request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), dcv_check_parameters=DcvCheckParameters( - validation_details=DcvAcmeDns01ValidationDetails(key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo") - ) + validation_details=DcvAcmeDns01ValidationDetails( + key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" + ) + ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) assert response.status_code == 200 mpic_response = self.mpic_response_adapter.validate_json(response.text) - + assert mpic_response.is_valid is True + # fmt: off @pytest.mark.parametrize('domain_or_ip_target, purpose_of_test', [ ('dns-01-leading-whitespace.integration-testing.open-mpic.org', 'leading whitespace'), ('dns-01-trailing-whitespace.integration-testing.open-mpic.org', 'trailing'), ('dns-01-nxdomain.integration-testing.open-mpic.org', 'NXDOMAIN') ]) - def api_should_return_200_is_valid_false_given_invalid_dns_01_validation(self, api_client, domain_or_ip_target, purpose_of_test): + # fmt: on + def api_should_return_200_is_valid_false_given_invalid_dns_01_validation( + self, api_client, domain_or_ip_target, purpose_of_test + ): print(f"Running test for {domain_or_ip_target} ({purpose_of_test})") request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), dcv_check_parameters=DcvCheckParameters( - validation_details=DcvAcmeDns01ValidationDetails(key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo") - ) + validation_details=DcvAcmeDns01ValidationDetails( + key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" + ) + ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) assert response.status_code == 200 mpic_response = self.mpic_response_adapter.validate_json(response.text) - + assert mpic_response.is_valid is False + # fmt: off @pytest.mark.parametrize('domain_or_ip_target, dns_record_type, challenge_value, purpose_of_test', [ ('dns-change-txt.integration-testing.open-mpic.org', DnsRecordType.TXT, "1234567890abcdefg.", 'standard TXT dns change'), ('dns-change-cname.integration-testing.open-mpic.org', DnsRecordType.CNAME, "1234567890abcdefg.", 'standard CNAME dns change'), ('dns-change-caa.integration-testing.open-mpic.org', DnsRecordType.CAA, '0 dnschange "1234567890abcdefg."', 'standard CAA dns change'), ]) - def api_should_return_200_is_valid_true_given_valid_dns_change_validation(self, api_client, domain_or_ip_target, dns_record_type, challenge_value, purpose_of_test): + # fmt: on + def api_should_return_200_is_valid_true_given_valid_dns_change_validation( + self, api_client, domain_or_ip_target, dns_record_type, challenge_value, purpose_of_test + ): print(f"Running test for {domain_or_ip_target} ({purpose_of_test})") request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), dcv_check_parameters=DcvCheckParameters( - validation_details=DcvDnsChangeValidationDetails(challenge_value=challenge_value, dns_record_type=dns_record_type, dns_name_prefix="") - ) + validation_details=DcvDnsChangeValidationDetails( + challenge_value=challenge_value, dns_record_type=dns_record_type, dns_name_prefix="" + ) + ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body @@ -215,11 +249,10 @@ def api_should_return_200_is_valid_true_given_valid_dns_change_validation(self, def api_should_return_200_and_failed_corroboration_given_failed_dcv_check(self, api_client): request = MpicDcvRequest( - domain_or_ip_target='ifconfig.me', + domain_or_ip_target="ifconfig.me", dcv_check_parameters=DcvCheckParameters( - validation_details=DcvWebsiteChangeValidationDetails(http_token_path='/', - challenge_value='test') - ) + validation_details=DcvWebsiteChangeValidationDetails(http_token_path="/", challenge_value="test") + ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body @@ -230,9 +263,13 @@ def api_should_return_200_and_failed_corroboration_given_failed_dcv_check(self, def api_should_return_400_given_invalid_orchestration_parameters_in_request(self, api_client): request = MpicCaaRequest( - domain_or_ip_target='example.com', - orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=5), # invalid quorum count - caa_check_parameters=CaaCheckParameters(certificate_type=CertificateType.TLS_SERVER, caa_domains=['mozilla.com']) + domain_or_ip_target="example.com", + orchestration_parameters=MpicRequestOrchestrationParameters( + perspective_count=3, quorum_count=5 + ), # invalid quorum count + caa_check_parameters=CaaCheckParameters( + certificate_type=CertificateType.TLS_SERVER, caa_domains=["mozilla.com"] + ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body @@ -240,20 +277,25 @@ def api_should_return_400_given_invalid_orchestration_parameters_in_request(self assert response.status_code == 400 response_body = json.loads(response.text) print("\nResponse:\n", json.dumps(response_body, indent=4)) # pretty print response body - assert response_body['error'] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key - assert any(issue['issue_type'] == MpicRequestValidationMessages.INVALID_QUORUM_COUNT.key for issue in response_body['validation_issues']) + assert response_body["error"] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key + assert any( + issue["issue_type"] == MpicRequestValidationMessages.INVALID_QUORUM_COUNT.key + for issue in response_body["validation_issues"] + ) def api_should_return_400_given_invalid_check_type_in_request(self, api_client): request = MpicCaaRequest( - domain_or_ip_target='example.com', + domain_or_ip_target="example.com", orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - caa_check_parameters=CaaCheckParameters(certificate_type=CertificateType.TLS_SERVER, caa_domains=['mozilla.com']) + caa_check_parameters=CaaCheckParameters( + certificate_type=CertificateType.TLS_SERVER, caa_domains=["mozilla.com"] + ), ) - request.check_type = 'invalid_check_type' + request.check_type = "invalid_check_type" print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) assert response.status_code == 400 response_body = json.loads(response.text) print("\nResponse:\n", json.dumps(response_body, indent=4)) - assert response_body['error'] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key + assert response_body["error"] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key diff --git a/tests/integration/testing_api_client.py b/tests/integration/testing_api_client.py index bad25d3..edfe0fe 100644 --- a/tests/integration/testing_api_client.py +++ b/tests/integration/testing_api_client.py @@ -8,19 +8,16 @@ class TestingApiClient: def __init__(self): self.service_base_url = get_api_url.extract_api_url(None) self.api_key = get_api_key.extract_api_key(None) - print('\nURL: ', self.service_base_url) - print('\nAPI Key: ', self.api_key) + print("\nURL: ", self.service_base_url) + print("\nAPI Key: ", self.api_key) self._session = requests.Session() def get(self, url_suffix): - return self._session.get(self.service_base_url + '/' + url_suffix) + return self._session.get(self.service_base_url + "/" + url_suffix) def post(self, url_suffix, data): - headers = { - 'content-type': 'application/json', - 'x-api-key': self.api_key - } - response = self._session.post(self.service_base_url + '/' + url_suffix, headers=headers, data=data) + headers = {"content-type": "application/json", "x-api-key": self.api_key} + response = self._session.post(self.service_base_url + "/" + url_suffix, headers=headers, data=data) return response def close(self): diff --git a/tests/unit/aws_lambda_mpic/conftest.py b/tests/unit/aws_lambda_mpic/conftest.py index a113734..00bd34e 100644 --- a/tests/unit/aws_lambda_mpic/conftest.py +++ b/tests/unit/aws_lambda_mpic/conftest.py @@ -13,7 +13,7 @@ def setup_logging(): log_output = StringIO() # to be able to inspect what gets logged handler = logging.StreamHandler(log_output) - handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) # Configure fresh logging logging.basicConfig(handlers=[handler]) diff --git a/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py index 9fdcc55..dfe7379 100644 --- a/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py @@ -3,22 +3,18 @@ import dns import pytest -import aws_lambda_mpic.mpic_caa_checker_lambda.mpic_caa_checker_lambda_function as mpic_caa_checker_lambda_function -from open_mpic_core.common_domain.check_response import CaaCheckResponse, CaaCheckResponseDetails +from open_mpic_core import CaaCheckResponse, CaaCheckResponseDetails from open_mpic_core_test.test_util.mock_dns_object_creator import MockDnsObjectCreator from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator +import aws_lambda_mpic.mpic_caa_checker_lambda.mpic_caa_checker_lambda_function as mpic_caa_checker_lambda_function # noinspection PyMethodMayBeStatic class TestCaaCheckerLambda: @staticmethod - @pytest.fixture(scope='class') + @pytest.fixture(scope="class") def set_env_variables(): - envvars = { - 'AWS_REGION': 'us-east-1', - 'default_caa_domains': 'ca1.com|ca2.org|ca3.net', - 'log_level': 'TRACE' - } + envvars = {"AWS_REGION": "us-east-1", "default_caa_domains": "ca1.com|ca2.org|ca3.net", "log_level": "TRACE"} with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: for k, v in envvars.items(): class_scoped_monkeypatch.setenv(k, v) @@ -28,11 +24,11 @@ def set_env_variables(): def lambda_handler__should_do_caa_check_using_configured_caa_checker(self, set_env_variables, mocker): mock_caa_result = TestCaaCheckerLambda.create_caa_check_response() mock_return_value = { - 'statusCode': 200, # note: must be snakeCase - 'headers': {'Content-Type': 'application/json'}, - 'body': mock_caa_result.model_dump_json() + "statusCode": 200, # note: must be snakeCase + "headers": {"Content-Type": "application/json"}, + "body": mock_caa_result.model_dump_json(), } - mocker.patch('open_mpic_core.mpic_caa_checker.mpic_caa_checker.MpicCaaChecker.check_caa', return_value=mock_caa_result) + mocker.patch("open_mpic_core.MpicCaaChecker.check_caa", return_value=mock_caa_result) caa_check_request = ValidCheckCreator.create_valid_caa_check_request() result = mpic_caa_checker_lambda_function.lambda_handler(caa_check_request, None) assert result == mock_return_value @@ -40,25 +36,26 @@ def lambda_handler__should_do_caa_check_using_configured_caa_checker(self, set_e def lambda_handler__should_set_log_level_of_caa_checker(self, set_env_variables, setup_logging, mocker): caa_check_request = ValidCheckCreator.create_valid_caa_check_request() - records = [MockDnsObjectCreator.create_caa_record(0, 'issue', 'ca1.org')] + records = [MockDnsObjectCreator.create_caa_record(0, "issue", "ca1.org")] mock_rrset = MockDnsObjectCreator.create_rrset(dns.rdatatype.CAA, *records) mock_domain = dns.name.from_text(caa_check_request.domain_or_ip_target) mock_return = (mock_rrset, mock_domain) - mocker.patch('open_mpic_core.mpic_caa_checker.mpic_caa_checker.MpicCaaChecker.find_caa_records_and_domain', - return_value=mock_return) + mocker.patch("open_mpic_core.MpicCaaChecker.find_caa_records_and_domain", return_value=mock_return) result = mpic_caa_checker_lambda_function.lambda_handler(caa_check_request, None) - assert result['statusCode'] == 200 + assert result["statusCode"] == 200 log_contents = setup_logging.getvalue() - assert all(text in log_contents for text in ['MpicCaaChecker', 'TRACE']) # Verify the log level was set + assert all(text in log_contents for text in ["MpicCaaChecker", "TRACE"]) # Verify the log level was set @staticmethod def create_caa_check_response(): - return CaaCheckResponse(perspective_code='us-east-1', check_passed=True, - details=CaaCheckResponseDetails(caa_record_present=True, - found_at='example.com'), - timestamp_ns=time.time_ns()) + return CaaCheckResponse( + perspective_code="us-east-1", + check_passed=True, + details=CaaCheckResponseDetails(caa_record_present=True, found_at="example.com"), + timestamp_ns=time.time_ns(), + ) -if __name__ == '__main__': +if __name__ == "__main__": pytest.main() diff --git a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py index c21705f..11c36be 100644 --- a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py @@ -1,25 +1,22 @@ import time import pytest +from open_mpic_core import MpicValidationError +from open_mpic_core import DcvHttpCheckResponseDetails +from open_mpic_core import DcvValidationMethod +from open_mpic_core import DcvCheckResponse import aws_lambda_mpic.mpic_dcv_checker_lambda.mpic_dcv_checker_lambda_function as mpic_dcv_checker_lambda_function -from open_mpic_core.common_domain.validation_error import MpicValidationError -from open_mpic_core.common_domain.check_response_details import DcvHttpCheckResponseDetails -from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod -from open_mpic_core.common_domain.check_response import DcvCheckResponse -from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator +from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator from unit.aws_lambda_mpic.conftest import setup_logging # noinspection PyMethodMayBeStatic class TestDcvCheckerLambda: @staticmethod - @pytest.fixture(scope='class') + @pytest.fixture(scope="class") def set_env_variables(): - envvars = { - 'AWS_REGION': 'us-east-1', - 'log_level': 'TRACE' - } + envvars = {"AWS_REGION": "us-east-1", "log_level": "TRACE"} with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: for k, v in envvars.items(): class_scoped_monkeypatch.setenv(k, v) @@ -29,49 +26,57 @@ def set_env_variables(): def lambda_handler__should_do_dcv_check_using_configured_dcv_checker(self, set_env_variables, mocker): mock_dcv_response = TestDcvCheckerLambda.create_dcv_check_response() mock_return_value = { - 'statusCode': 200, # note: must be snakeCase - 'headers': {'Content-Type': 'application/json'}, - 'body': mock_dcv_response.model_dump_json() + "statusCode": 200, # note: must be snakeCase + "headers": {"Content-Type": "application/json"}, + "body": mock_dcv_response.model_dump_json(), } - mocker.patch('open_mpic_core.mpic_dcv_checker.mpic_dcv_checker.MpicDcvChecker.check_dcv', return_value=mock_dcv_response) + mocker.patch("open_mpic_core.MpicDcvChecker.check_dcv", return_value=mock_dcv_response) dcv_check_request = ValidCheckCreator.create_valid_http_check_request() result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None) assert result == mock_return_value + # fmt: off @pytest.mark.parametrize('error_type, error_message, expected_status_code', [ ('404', 'Not Found', 404), ('No Answer', 'The DNS response does not contain an answer to the question', 500) ]) + # fmt: on def lambda_handler__should_return_appropriate_status_code_given_errors_in_response( - self, error_type: str, error_message: str, expected_status_code: int, set_env_variables, mocker): + self, error_type: str, error_message: str, expected_status_code: int, set_env_variables, mocker + ): mock_dcv_response = TestDcvCheckerLambda.create_dcv_check_response() mock_dcv_response.check_passed = False mock_dcv_response.errors = [(MpicValidationError(error_type=error_type, error_message=error_message))] mock_return_value = { - 'statusCode': expected_status_code, - 'headers': {'Content-Type': 'application/json'}, - 'body': mock_dcv_response.model_dump_json() + "statusCode": expected_status_code, + "headers": {"Content-Type": "application/json"}, + "body": mock_dcv_response.model_dump_json(), } - mocker.patch('open_mpic_core.mpic_dcv_checker.mpic_dcv_checker.MpicDcvChecker.check_dcv', return_value=mock_dcv_response) + mocker.patch("open_mpic_core.MpicDcvChecker.check_dcv", return_value=mock_dcv_response) dcv_check_request = ValidCheckCreator.create_valid_http_check_request() result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None) assert result == mock_return_value def lambda_handler__should_set_log_level_of_dcv_checker(self, set_env_variables, mocker, setup_logging): dcv_check_request = ValidCheckCreator.create_valid_http_check_request() - mocker.patch('open_mpic_core.mpic_dcv_checker.mpic_dcv_checker.MpicDcvChecker.perform_http_based_validation', - return_value=TestDcvCheckerLambda.create_dcv_check_response()) + mocker.patch( + "open_mpic_core.MpicDcvChecker.perform_http_based_validation", + return_value=TestDcvCheckerLambda.create_dcv_check_response(), + ) result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None) - assert result['statusCode'] == 200 + assert result["statusCode"] == 200 log_contents = setup_logging.getvalue() - assert all(text in log_contents for text in ['MpicDcvChecker', 'TRACE']) # Verify the log level was set + assert all(text in log_contents for text in ["MpicDcvChecker", "TRACE"]) # Verify the log level was set @staticmethod def create_dcv_check_response(): - return DcvCheckResponse(perspective_code='us-east-1', check_passed=True, - details=DcvHttpCheckResponseDetails(validation_method=DcvValidationMethod.WEBSITE_CHANGE_V2), - timestamp_ns=time.time_ns()) + return DcvCheckResponse( + perspective_code="us-east-1", + check_passed=True, + details=DcvHttpCheckResponseDetails(validation_method=DcvValidationMethod.WEBSITE_CHANGE_V2), + timestamp_ns=time.time_ns(), + ) -if __name__ == '__main__': +if __name__ == "__main__": pytest.main() diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index 46e888f..e1fa801 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -6,22 +6,27 @@ import pytest import yaml -from aws_lambda_powertools.utilities.parser.models import APIGatewayProxyEventModel, APIGatewayEventRequestContext, \ - APIGatewayEventIdentity -from open_mpic_core.mpic_coordinator.domain.remote_perspective import RemotePerspective +from aws_lambda_powertools.utilities.parser.models import ( + APIGatewayProxyEventModel, + APIGatewayEventRequestContext, + APIGatewayEventIdentity, +) from pydantic import TypeAdapter -from open_mpic_core.common_domain.check_request import DcvCheckRequest -from open_mpic_core.common_domain.check_response import DcvCheckResponse, CaaCheckResponse -from open_mpic_core.common_domain.enum.check_type import CheckType -from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails, CaaCheckResponseDetails -from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod -from open_mpic_core.mpic_coordinator.domain.mpic_orchestration_parameters import MpicEffectiveOrchestrationParameters -from open_mpic_core.mpic_coordinator.domain.mpic_response import MpicCaaResponse -from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import MpicCoordinatorLambdaHandler +from open_mpic_core import RemotePerspective +from open_mpic_core import DcvCheckRequest, DcvCheckResponse, CaaCheckResponse +from open_mpic_core import CheckType +from open_mpic_core import DcvDnsCheckResponseDetails, CaaCheckResponseDetails +from open_mpic_core import DcvValidationMethod +from open_mpic_core import MpicEffectiveOrchestrationParameters +from open_mpic_core import MpicCaaResponse from botocore.response import StreamingBody import aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function as mpic_coordinator_lambda_function -from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import PerspectiveEndpoints, PerspectiveEndpointInfo +from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import MpicCoordinatorLambdaHandler +from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import ( + PerspectiveEndpoints, + PerspectiveEndpointInfo, +) from open_mpic_core_test.test_util.valid_mpic_request_creator import ValidMpicRequestCreator from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator @@ -30,34 +35,49 @@ # noinspection PyMethodMayBeStatic class TestMpicCoordinatorLambda: @staticmethod - @pytest.fixture(scope='class') + @pytest.fixture(scope="class") def set_env_variables(): perspectives_as_dict = { - "us-east-1": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-east-1:123456789012:caa/us-east-1'), - dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-east-1:123456789012:dcv/us-east-1')), - "us-west-1": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-west-1:123456789012:caa/us-west-1'), - dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:us-west-1:123456789012:dcv/us-west-1')), - "eu-west-2": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-west-2:123456789012:caa/eu-west-2'), - dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-west-2:123456789012:dcv/eu-west-2')), - "eu-central-2": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-central-2:123456789012:caa/eu-central-2'), - dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:eu-central-2:123456789012:dcv/eu-central-2')), - "ap-northeast-1": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-northeast-1:123456789012:caa/ap-northeast-1'), - dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-northeast-1:123456789012:dcv/ap-northeast-1')), - "ap-south-2": PerspectiveEndpoints(caa_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-south-2:123456789012:caa/ap-south-2'), - dcv_endpoint_info=PerspectiveEndpointInfo(arn='arn:aws:acm-pca:ap-south-2:123456789012:dcv/ap-south-2')) + "us-east-1": PerspectiveEndpoints( + caa_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:us-east-1:123:caa/us-east-1"), + dcv_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:us-east-1:123:dcv/us-east-1"), + ), + "us-west-1": PerspectiveEndpoints( + caa_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:us-west-1:123:caa/us-west-1"), + dcv_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:us-west-1:123:dcv/us-west-1"), + ), + "eu-west-2": PerspectiveEndpoints( + caa_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:eu-west-2:123:caa/eu-west-2"), + dcv_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:eu-west-2:123:dcv/eu-west-2"), + ), + "eu-central-2": PerspectiveEndpoints( + caa_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:eu-central-2:123:caa/eu-central-2"), + dcv_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:eu-central-2:123:dcv/eu-central-2"), + ), + "ap-northeast-1": PerspectiveEndpoints( + caa_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:ap-northeast-1:123:caa/ap-northeast-1"), + dcv_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:ap-northeast-1:123:dcv/ap-northeast-1"), + ), + "ap-south-2": PerspectiveEndpoints( + caa_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:ap-south-2:123:caa/ap-south-2"), + dcv_endpoint_info=PerspectiveEndpointInfo(arn="arn:aws:ap-south-2:123:dcv/ap-south-2"), + ), } + # fmt: on envvars = { - 'perspectives': json.dumps({k: v.model_dump() for k, v in perspectives_as_dict.items()}), - 'default_perspective_count': '3', - 'hash_secret': 'test_secret', - 'log_level': 'TRACE' + "perspectives": json.dumps({k: v.model_dump() for k, v in perspectives_as_dict.items()}), + "default_perspective_count": "3", + "hash_secret": "test_secret", + "log_level": "TRACE", } with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: for k, v in envvars.items(): class_scoped_monkeypatch.setenv(k, v) yield class_scoped_monkeypatch # restore the environment afterward - async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arguments_and_return_check_response(self, set_env_variables, mocker): + async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arguments_and_return_check_response( + self, set_env_variables, mocker + ): # Mock the aioboto3 client creation and context manager mock_client = AsyncMock() mock_client.invoke = AsyncMock(side_effect=self.create_successful_aioboto3_response_for_dcv_check) @@ -66,32 +86,29 @@ async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arg mock_client.__aenter__.return_value = mock_client # Mock the session creation and client initialization - mock_session = mocker.patch('aioboto3.Session') + mock_session = mocker.patch("aioboto3.Session") mock_session.return_value.client.return_value = mock_client - # mocker.patch('botocore.client.BaseClient._make_api_call', side_effect=self.create_successful_boto3_api_call_response_for_dcv_check) dcv_check_request = ValidCheckCreator.create_valid_dns_check_request() - mpic_coordinator_lambda_handler = MpicCoordinatorLambdaHandler() + lambda_handler = MpicCoordinatorLambdaHandler() - await mpic_coordinator_lambda_handler.initialize_client_pools() + await lambda_handler.initialize_client_pools() - perspective_code = 'us-west-1' - check_response = await mpic_coordinator_lambda_handler.call_remote_perspective( - RemotePerspective(code=perspective_code, rir='arin'), - CheckType.DCV, - dcv_check_request + perspective_code = "us-west-1" + check_response = await lambda_handler.call_remote_perspective( + RemotePerspective(code=perspective_code, rir="arin"), CheckType.DCV, dcv_check_request ) assert check_response.check_passed is True # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call assert check_response.perspective_code == dcv_check_request.domain_or_ip_target - function_endpoint_info = mpic_coordinator_lambda_handler.remotes_per_perspective_per_check_type[CheckType.DCV][perspective_code] + function_endpoint_info = lambda_handler.remotes_per_perspective_per_check_type[CheckType.DCV][perspective_code] # Verify the mock was called correctly mock_client.invoke.assert_called_once_with( FunctionName=function_endpoint_info.arn, - InvocationType='RequestResponse', - Payload=dcv_check_request.model_dump_json() + InvocationType="RequestResponse", + Payload=dcv_check_request.model_dump_json(), ) def lambda_handler__should_return_400_error_and_details_given_invalid_request_body(self): @@ -102,20 +119,20 @@ def lambda_handler__should_return_400_error_and_details_given_invalid_request_bo api_request.body = request.model_dump_json() # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) - assert result['statusCode'] == 400 - result_body = json.loads(result['body']) - assert result_body['validation_issues'][0]['type'] == 'string_type' + assert result["statusCode"] == 400 + result_body = json.loads(result["body"]) + assert result_body["validation_issues"][0]["type"] == "string_type" def lambda_handler__should_return_400_error_and_details_given_invalid_check_type(self): request = ValidMpicRequestCreator.create_valid_dcv_mpic_request() - request.check_type = 'invalid_check_type' + request.check_type = "invalid_check_type" api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = request.model_dump_json() # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) - assert result['statusCode'] == 400 - result_body = json.loads(result['body']) - assert result_body['validation_issues'][0]['type'] == 'literal_error' + assert result["statusCode"] == 400 + result_body = json.loads(result["body"]) + assert result_body["validation_issues"][0]["type"] == "literal_error" def lambda_handler__should_return_400_error_given_logically_invalid_request(self, set_env_variables): request = ValidMpicRequestCreator.create_valid_dcv_mpic_request() @@ -124,31 +141,35 @@ def lambda_handler__should_return_400_error_given_logically_invalid_request(self api_request.body = request.model_dump_json() # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) - assert result['statusCode'] == 400 - result_body = json.loads(result['body']) - assert result_body['validation_issues'][0]['issue_type'] == 'invalid-perspective-count' + assert result["statusCode"] == 400 + result_body = json.loads(result["body"]) + assert result_body["validation_issues"][0]["issue_type"] == "invalid-perspective-count" def lambda_handler__should_return_500_error_given_other_unexpected_errors(self, set_env_variables, mocker): request = ValidMpicRequestCreator.create_valid_dcv_mpic_request() api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = request.model_dump_json() - mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.coordinate_mpic', - side_effect=Exception('Something went wrong')) + mocker.patch( + "open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.coordinate_mpic", + side_effect=Exception("Something went wrong"), + ) # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) - assert result['statusCode'] == 500 + assert result["statusCode"] == 500 def lambda_handler__should_coordinate_mpic_using_configured_mpic_coordinator(self, set_env_variables, mocker): mpic_request = ValidMpicRequestCreator.create_valid_mpic_request(CheckType.CAA) api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = mpic_request.model_dump_json() mock_return_value = TestMpicCoordinatorLambda.create_caa_mpic_response() - mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.coordinate_mpic', - return_value=mock_return_value) + mocker.patch( + "open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.coordinate_mpic", + return_value=mock_return_value, + ) expected_response = { - 'statusCode': 200, - 'headers': {'Content-Type': 'application/json'}, - 'body': mock_return_value.model_dump_json() + "statusCode": 200, + "headers": {"Content-Type": "application/json"}, + "body": mock_return_value.model_dump_json(), } # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) @@ -159,22 +180,22 @@ def lambda_handler__should_set_log_level_for_coordinator(self, set_env_variables api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = mpic_request.model_dump_json() mocked_perspective_responses = [ - CaaCheckResponse(perspective_code='us-east-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='us-west-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='eu-west-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='eu-central-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='ap-northeast-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='ap-south-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponse( + perspective_code=code, check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False) + ) + for code in ["us-east-1", "us-west-1", "eu-west-2", "eu-central-2", "ap-northeast-1", "ap-south-2"] ] - mocked_validity_per_perspective = {response.perspective_code: response.check_passed for response in mocked_perspective_responses} + mocked_validity_per_perspective = { + response.perspective_code: response.check_passed for response in mocked_perspective_responses + } mock_return = (mocked_perspective_responses, mocked_validity_per_perspective) - mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.issue_async_calls_and_collect_responses', return_value=mock_return) + mocker.patch("open_mpic_core.MpicCoordinator.issue_async_calls_and_collect_responses", return_value=mock_return) # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) - assert result['statusCode'] == 200 + assert result["statusCode"] == 200 log_contents = setup_logging.getvalue() - assert all(text in log_contents for text in ['MpicCoordinator', 'TRACE']) # Verify the log level was set + assert all(text in log_contents for text in ["MpicCoordinator", "TRACE"]) # Verify the log level was set def load_aws_region_config__should_return_dict_of_aws_regions_with_proximity_info_by_region_code(self): mpic_coordinator_lambda_handler = MpicCoordinatorLambdaHandler() @@ -182,8 +203,8 @@ def load_aws_region_config__should_return_dict_of_aws_regions_with_proximity_inf all_possible_perspectives = TestMpicCoordinatorLambda.get_perspectives_by_code_dict_from_file() assert len(loaded_aws_regions.keys()) == len(all_possible_perspectives.keys()) # for example, us-east-1 is too close to us-east-2 - assert 'us-east-2' in loaded_aws_regions['us-east-1'].too_close_codes - assert 'us-east-1' in loaded_aws_regions['us-east-2'].too_close_codes + assert "us-east-2" in loaded_aws_regions["us-east-1"].too_close_codes + assert "us-east-1" in loaded_aws_regions["us-east-2"].too_close_codes def constructor__should_initialize_mpic_coordinator_and_set_target_perspectives(self, set_env_variables): mpic_coordinator_lambda_handler = MpicCoordinatorLambdaHandler() @@ -193,35 +214,42 @@ def constructor__should_initialize_mpic_coordinator_and_set_target_perspectives( mpic_coordinator = mpic_coordinator_lambda_handler.mpic_coordinator assert len(mpic_coordinator.target_perspectives) == 6 assert mpic_coordinator.default_perspective_count == 3 - assert mpic_coordinator.hash_secret == 'test_secret' + assert mpic_coordinator.hash_secret == "test_secret" # noinspection PyUnusedLocal def create_successful_boto3_api_call_response_for_dcv_check(self, lambda_method, lambda_configuration): - check_request = DcvCheckRequest.model_validate_json(lambda_configuration['Payload']) + check_request = DcvCheckRequest.model_validate_json(lambda_configuration["Payload"]) # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call - expected_response_body = DcvCheckResponse(perspective_code=check_request.domain_or_ip_target, - check_passed=True, details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01)) - expected_response = {'statusCode': 200, 'body': expected_response_body.model_dump_json()} - json_bytes = json.dumps(expected_response).encode('utf-8') + expected_response_body = DcvCheckResponse( + perspective_code=check_request.domain_or_ip_target, + check_passed=True, + details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01), + ) + expected_response = {"statusCode": 200, "body": expected_response_body.model_dump_json()} + json_bytes = json.dumps(expected_response).encode("utf-8") file_like_response = io.BytesIO(json_bytes) streaming_body_response = StreamingBody(file_like_response, len(json_bytes)) - return {'Payload': streaming_body_response} + return {"Payload": streaming_body_response} # noinspection PyUnusedLocal async def create_successful_aioboto3_response_for_dcv_check(self, *args, **kwargs): - check_request = DcvCheckRequest.model_validate_json(kwargs['Payload']) + check_request = DcvCheckRequest.model_validate_json(kwargs["Payload"]) # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call - expected_response_body = DcvCheckResponse(perspective_code=check_request.domain_or_ip_target, - check_passed=True, details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01)) - expected_response = {'statusCode': 200, 'body': expected_response_body.model_dump_json()} - json_bytes = json.dumps(expected_response).encode('utf-8') + expected_response_body = DcvCheckResponse( + perspective_code=check_request.domain_or_ip_target, + check_passed=True, + details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01), + ) + expected_response = {"statusCode": 200, "body": expected_response_body.model_dump_json()} + json_bytes = json.dumps(expected_response).encode("utf-8") # Mock the response structure that aioboto3 would return class MockStreamingBody: # noinspection PyMethodMayBeStatic async def read(self): return json_bytes - return {'Payload': MockStreamingBody()} + + return {"Payload": MockStreamingBody()} @staticmethod def create_caa_mpic_response(): @@ -233,13 +261,14 @@ def create_caa_mpic_response(): ), is_valid=True, perspectives=[], - caa_check_parameters=caa_request.caa_check_parameters + caa_check_parameters=caa_request.caa_check_parameters, ) + # noinspection PyUnusedLocal @staticmethod def create_caa_perspective_response(*args, **kwargs) -> CaaCheckResponse: return CaaCheckResponse( - perspective_code=kwargs['perspective'].code, + perspective_code=kwargs["perspective"].code, check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False), ) @@ -247,27 +276,35 @@ def create_caa_perspective_response(*args, **kwargs) -> CaaCheckResponse: @staticmethod def create_api_gateway_request(): request = APIGatewayProxyEventModel( - resource='whatever', path='/mpic', - httpMethod='POST', headers={'Content-Type': 'application/json'}, multiValueHeaders={}, + resource="whatever", + path="/mpic", + httpMethod="POST", + headers={"Content-Type": "application/json"}, + multiValueHeaders={}, requestContext=APIGatewayEventRequestContext( - accountId='whatever', apiId='whatever', stage='whatever', protocol='whatever', - identity=APIGatewayEventIdentity( - sourceIp='test-invoke-source-ip' - ), - requestId='whatever', requestTime='whatever', requestTimeEpoch=datetime.now(), - resourcePath='whatever', httpMethod='POST', path='/mpic' + accountId="whatever", + apiId="whatever", + stage="whatever", + protocol="whatever", + identity=APIGatewayEventIdentity(sourceIp="test-invoke-source-ip"), + requestId="whatever", + requestTime="whatever", + requestTimeEpoch=datetime.now(), + resourcePath="whatever", + httpMethod="POST", + path="/mpic", ), ) return request @staticmethod def get_perspectives_by_code_dict_from_file() -> dict[str, RemotePerspective]: - with resources.files('resources').joinpath('aws_region_config.yaml').open('r') as file: + with resources.files("resources").joinpath("aws_region_config.yaml").open("r") as file: perspectives_yaml = yaml.safe_load(file) perspective_type_adapter = TypeAdapter(list[RemotePerspective]) - perspectives = perspective_type_adapter.validate_python(perspectives_yaml['aws_available_regions']) + perspectives = perspective_type_adapter.validate_python(perspectives_yaml["aws_available_regions"]) return {perspective.code: perspective for perspective in perspectives} -if __name__ == '__main__': +if __name__ == "__main__": pytest.main() From 8e704ad478361742e87b549ad89341c32b6d97da Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Fri, 31 Jan 2025 01:40:01 -0500 Subject: [PATCH 02/10] bumped dependency version. removed unnecessary print statement for an error --- pyproject.toml | 2 +- src/aws_lambda_mpic/__about__.py | 2 +- .../mpic_coordinator_lambda/mpic_coordinator_lambda_function.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9a6887c..7b4c0de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "pydantic==2.8.2", "aiohttp==3.11.11", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==4.6.2", + "open-mpic-core==4.7.1", "aioboto3~=13.3.0", "black==24.8.0", ] diff --git a/src/aws_lambda_mpic/__about__.py b/src/aws_lambda_mpic/__about__.py index 6a9beea..3d26edf 100644 --- a/src/aws_lambda_mpic/__about__.py +++ b/src/aws_lambda_mpic/__about__.py @@ -1 +1 @@ -__version__ = "0.4.0" +__version__ = "0.4.1" diff --git a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py index 0184468..c45f1d6 100644 --- a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py @@ -206,7 +206,6 @@ def wrapper(*args, **kwargs): except Exception as e: logger.error(f"An error occurred: {str(e)}") print(traceback.format_exc()) - print(f"BOY HOWDY error occurred: {str(e)}") return { "statusCode": 500, "headers": {"Content-Type": "application/json"}, From c85eeb103ce82ea5fa01c3a387fb3f4770bb6e19 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Fri, 31 Jan 2025 12:02:33 -0500 Subject: [PATCH 03/10] updated core version again --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7b4c0de..e0b8b47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "pydantic==2.8.2", "aiohttp==3.11.11", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==4.7.1", + "open-mpic-core==4.7.2", "aioboto3~=13.3.0", "black==24.8.0", ] From 231c403a999f21733cbe5224311decf50b2b4f49 Mon Sep 17 00:00:00 2001 From: Henry Birge-Lee Date: Sun, 2 Feb 2025 18:26:42 -0500 Subject: [PATCH 04/10] get remote perspective name from call --- deploy.sh | 2 +- pyproject.toml | 4 ++-- .../mpic_caa_checker_lambda_function.py | 2 -- .../mpic_dcv_checker_lambda_function.py | 4 +--- .../test_mpic_coordinator_lambda.py | 19 ++++++++++--------- 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/deploy.sh b/deploy.sh index 3d0de2b..1d73363 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,2 +1,2 @@ #!/bin/bash -./clean.sh; hatch run lambda-layer:install; cd layer; ./package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -var="dnssec_enabled=false" -auto-approve; cd .. +./clean.sh; hatch env prune; hatch run lambda-layer:install; cd layer; ./package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -var="dnssec_enabled=false" -auto-approve; cd .. diff --git a/pyproject.toml b/pyproject.toml index ae7f586..f120b92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,14 +26,14 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - #"open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@ds-trace-logging", + "open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@birgelee-dcv-caa-perspective-code-response", "pyyaml==6.0.1", "requests>=2.32.3", "dnspython==2.6.1", "pydantic==2.8.2", "aiohttp==3.11.11", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==4.6.1", + #"open-mpic-core==4.6.1", "aioboto3~=13.3.0", ] diff --git a/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py b/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py index 87df51d..808cfa3 100644 --- a/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py @@ -12,7 +12,6 @@ class MpicCaaCheckerLambdaHandler: def __init__(self): - self.perspective_code = os.environ['AWS_REGION'] self.default_caa_domain_list = os.environ['default_caa_domains'].split("|") self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None @@ -21,7 +20,6 @@ def __init__(self): self.logger.setLevel(self.log_level) self.caa_checker = MpicCaaChecker(default_caa_domain_list=self.default_caa_domain_list, - perspective_code=self.perspective_code, log_level=self.logger.level) def process_invocation(self, caa_request: CaaCheckRequest): diff --git a/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py b/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py index ef6cf1f..432738a 100644 --- a/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py @@ -12,15 +12,13 @@ class MpicDcvCheckerLambdaHandler: def __init__(self): - self.perspective_code = os.environ['AWS_REGION'] self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None self.logger = logger.getChild(self.__class__.__name__) if self.log_level: self.logger.setLevel(self.log_level) - self.dcv_checker = MpicDcvChecker(perspective_code=self.perspective_code, - reuse_http_client=False, + self.dcv_checker = MpicDcvChecker(reuse_http_client=False, log_level=self.logger.level) def process_invocation(self, dcv_request: DcvCheckRequest): diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index 46e888f..ce5a96a 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -12,7 +12,7 @@ from pydantic import TypeAdapter from open_mpic_core.common_domain.check_request import DcvCheckRequest -from open_mpic_core.common_domain.check_response import DcvCheckResponse, CaaCheckResponse +from open_mpic_core.common_domain.check_response import DcvCheckResponse, CaaCheckResponse, DcvCheckResponseWithPerspectiveCode, CaaCheckResponseWithPerspectiveCode from open_mpic_core.common_domain.enum.check_type import CheckType from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails, CaaCheckResponseDetails from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod @@ -57,6 +57,7 @@ def set_env_variables(): class_scoped_monkeypatch.setenv(k, v) yield class_scoped_monkeypatch # restore the environment afterward + @pytest.mark.skip("Perspective names are no longer returned. Value hijacking trick is not valid anymore.") async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arguments_and_return_check_response(self, set_env_variables, mocker): # Mock the aioboto3 client creation and context manager mock_client = AsyncMock() @@ -83,7 +84,7 @@ async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arg ) assert check_response.check_passed is True # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call - assert check_response.perspective_code == dcv_check_request.domain_or_ip_target + assert check_response.perspective == dcv_check_request.domain_or_ip_target function_endpoint_info = mpic_coordinator_lambda_handler.remotes_per_perspective_per_check_type[CheckType.DCV][perspective_code] @@ -159,14 +160,14 @@ def lambda_handler__should_set_log_level_for_coordinator(self, set_env_variables api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = mpic_request.model_dump_json() mocked_perspective_responses = [ - CaaCheckResponse(perspective_code='us-east-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='us-west-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='eu-west-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='eu-central-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='ap-northeast-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), - CaaCheckResponse(perspective_code='ap-south-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponseWithPerspectiveCode(perspective='us-east-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponseWithPerspectiveCode(perspective='us-west-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponseWithPerspectiveCode(perspective='eu-west-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponseWithPerspectiveCode(perspective='eu-central-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponseWithPerspectiveCode(perspective='ap-northeast-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponseWithPerspectiveCode(perspective='ap-south-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), ] - mocked_validity_per_perspective = {response.perspective_code: response.check_passed for response in mocked_perspective_responses} + mocked_validity_per_perspective = {response.perspective: response.check_passed for response in mocked_perspective_responses} mock_return = (mocked_perspective_responses, mocked_validity_per_perspective) mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.issue_async_calls_and_collect_responses', return_value=mock_return) From 96fff41990c207c92025b8ffea86a97322968ac5 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 4 Feb 2025 11:41:04 -0500 Subject: [PATCH 05/10] tweaked unit tests so they pass --- tests/integration/test_deployed_mpic_api.py | 36 +++++++------------ .../test_dcv_checker_lambda.py | 3 +- .../test_mpic_coordinator_lambda.py | 30 ++++++++-------- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index 8bba2fa..4f9e0a8 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -6,9 +6,9 @@ from open_mpic_core import ( CaaCheckParameters, - DcvWebsiteChangeValidationDetails, - DcvAcmeDns01ValidationDetails, - DcvDnsChangeValidationDetails, + DcvWebsiteChangeValidationParameters, + DcvAcmeDns01ValidationParameters, + DcvDnsChangeValidationParameters, ) from open_mpic_core import DcvCheckParameters from open_mpic_core import CertificateType, CheckType, DnsRecordType @@ -178,10 +178,8 @@ def api_should_return_200_given_valid_dns_01_validation(self, api_client, domain request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - dcv_check_parameters=DcvCheckParameters( - validation_details=DcvAcmeDns01ValidationDetails( - key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" - ) + dcv_check_parameters=DcvAcmeDns01ValidationParameters( + key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" ), ) @@ -206,10 +204,8 @@ def api_should_return_200_is_valid_false_given_invalid_dns_01_validation( request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - dcv_check_parameters=DcvCheckParameters( - validation_details=DcvAcmeDns01ValidationDetails( - key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" - ) + dcv_check_parameters=DcvAcmeDns01ValidationParameters( + key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" ), ) @@ -255,10 +251,8 @@ def api_should_return_200_given_invalid_http_01_validation( request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - dcv_check_parameters=DcvCheckParameters( - validation_details=DcvAcmeDns01ValidationDetails( - key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" - ) + dcv_check_parameters=DcvAcmeDns01ValidationParameters( + key_authorization="7FwkJPsKf-TH54wu4eiIFA3nhzYaevsL7953ihy-tpo" ), ) @@ -283,7 +277,7 @@ def api_should_return_200_given_valid_website_change_validation( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), dcv_check_parameters=DcvCheckParameters( - validation_details=DcvWebsiteChangeValidationDetails( + validation_details=DcvWebsiteChangeValidationParameters( http_token_path=http_token_path, challenge_value=challenge_value ) ), @@ -309,10 +303,8 @@ def api_should_return_200_is_valid_true_given_valid_dns_change_validation( request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - dcv_check_parameters=DcvCheckParameters( - validation_details=DcvDnsChangeValidationDetails( - challenge_value=challenge_value, dns_record_type=dns_record_type, dns_name_prefix="" - ) + dcv_check_parameters=DcvDnsChangeValidationParameters( + challenge_value=challenge_value, dns_record_type=dns_record_type, dns_name_prefix="" ), ) @@ -326,9 +318,7 @@ def api_should_return_200_is_valid_true_given_valid_dns_change_validation( def api_should_return_200_and_failed_corroboration_given_failed_dcv_check(self, api_client): request = MpicDcvRequest( domain_or_ip_target="ifconfig.me", - dcv_check_parameters=DcvCheckParameters( - validation_details=DcvWebsiteChangeValidationDetails(http_token_path="/", challenge_value="test") - ), + dcv_check_parameters=DcvWebsiteChangeValidationParameters(http_token_path="/", challenge_value="test") ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body diff --git a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py index 11c36be..682aae9 100644 --- a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py @@ -71,9 +71,8 @@ def lambda_handler__should_set_log_level_of_dcv_checker(self, set_env_variables, @staticmethod def create_dcv_check_response(): return DcvCheckResponse( - perspective_code="us-east-1", check_passed=True, - details=DcvHttpCheckResponseDetails(validation_method=DcvValidationMethod.WEBSITE_CHANGE_V2), + details=DcvHttpCheckResponseDetails(validation_method=DcvValidationMethod.WEBSITE_CHANGE), timestamp_ns=time.time_ns(), ) diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index b1f092a..d3a4800 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -14,7 +14,7 @@ from pydantic import TypeAdapter from open_mpic_core import RemotePerspective -from open_mpic_core import DcvCheckRequest, DcvCheckResponse, CaaCheckResponse +from open_mpic_core import DcvCheckRequest, DcvCheckResponse, CaaCheckResponse, PerspectiveResponse from open_mpic_core import CheckType from open_mpic_core import DcvDnsCheckResponseDetails, CaaCheckResponseDetails from open_mpic_core import DcvValidationMethod @@ -75,7 +75,6 @@ def set_env_variables(): class_scoped_monkeypatch.setenv(k, v) yield class_scoped_monkeypatch # restore the environment afterward - @pytest.mark.skip("Perspective names are no longer returned. Value hijacking trick is not valid anymore.") async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arguments_and_return_check_response( self, set_env_variables, mocker ): @@ -100,8 +99,8 @@ async def call_remote_perspective__should_make_aws_lambda_call_with_provided_arg RemotePerspective(code=perspective_code, rir="arin"), CheckType.DCV, dcv_check_request ) assert check_response.check_passed is True - # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call - assert check_response.perspective == dcv_check_request.domain_or_ip_target + # hijacking the value of 'details.found_at' to verify that the right arguments got passed to the call + assert check_response.details.found_at == dcv_check_request.domain_or_ip_target function_endpoint_info = lambda_handler.remotes_per_perspective_per_check_type[CheckType.DCV][perspective_code] @@ -181,17 +180,17 @@ def lambda_handler__should_set_log_level_for_coordinator(self, set_env_variables api_request = TestMpicCoordinatorLambda.create_api_gateway_request() api_request.body = mpic_request.model_dump_json() mocked_perspective_responses = [ - CaaCheckResponse( - perspective_code=code, check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False) + PerspectiveResponse( + perspective_code=code, + check_response=CaaCheckResponse( + check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False) + ), ) for code in ["us-east-1", "us-west-1", "eu-west-2", "eu-central-2", "ap-northeast-1", "ap-south-2"] ] - mocked_validity_per_perspective = { - response.perspective_code: response.check_passed for response in mocked_perspective_responses - } - mock_return = (mocked_perspective_responses, mocked_validity_per_perspective) + mock_return = mocked_perspective_responses - mocker.patch("open_mpic_core.MpicCoordinator.issue_async_calls_and_collect_responses", return_value=mock_return) + mocker.patch("open_mpic_core.MpicCoordinator.call_checkers_and_collect_responses", return_value=mock_return) # noinspection PyTypeChecker result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result["statusCode"] == 200 @@ -222,7 +221,6 @@ def create_successful_boto3_api_call_response_for_dcv_check(self, lambda_method, check_request = DcvCheckRequest.model_validate_json(lambda_configuration["Payload"]) # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call expected_response_body = DcvCheckResponse( - perspective_code=check_request.domain_or_ip_target, check_passed=True, details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01), ) @@ -235,11 +233,12 @@ def create_successful_boto3_api_call_response_for_dcv_check(self, lambda_method, # noinspection PyUnusedLocal async def create_successful_aioboto3_response_for_dcv_check(self, *args, **kwargs): check_request = DcvCheckRequest.model_validate_json(kwargs["Payload"]) - # hijacking the value of 'perspective_code' to verify that the right arguments got passed to the call + # hijacking the value of 'found_at' to verify that the right arguments got passed to the call expected_response_body = DcvCheckResponse( - perspective_code=check_request.domain_or_ip_target, check_passed=True, - details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01), + details=DcvDnsCheckResponseDetails( + validation_method=DcvValidationMethod.ACME_DNS_01, found_at=check_request.domain_or_ip_target + ), ) expected_response = {"statusCode": 200, "body": expected_response_body.model_dump_json()} json_bytes = json.dumps(expected_response).encode("utf-8") @@ -269,7 +268,6 @@ def create_caa_mpic_response(): @staticmethod def create_caa_perspective_response(*args, **kwargs) -> CaaCheckResponse: return CaaCheckResponse( - perspective_code=kwargs["perspective"].code, check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False), ) From bb6085d76cd6a83853b91b563b4d8e5598ab7de0 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 4 Feb 2025 12:54:15 -0500 Subject: [PATCH 06/10] tweaked integration test setups to make them pass again --- tests/integration/test_deployed_mpic_api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index 4f9e0a8..71ad615 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -12,7 +12,7 @@ ) from open_mpic_core import DcvCheckParameters from open_mpic_core import CertificateType, CheckType, DnsRecordType -from open_mpic_core import MpicCaaRequest, MpicDcvRequest, MpicResponse +from open_mpic_core import MpicCaaRequest, MpicDcvRequest, MpicResponse, PerspectiveResponse from open_mpic_core import MpicRequestOrchestrationParameters from open_mpic_core import MpicRequestValidationMessages from open_mpic_core import DcvAcmeHttp01ValidationParameters @@ -56,10 +56,10 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se mpic_response = self.mpic_response_adapter.validate_json(response.text) print("\nResponse:\n", json.dumps(mpic_response.model_dump(), indent=4)) # pretty print response body assert mpic_response.is_valid is True - perspectives_list = mpic_response.perspectives + perspectives_list: list[PerspectiveResponse] = mpic_response.perspectives assert len(perspectives_list) == request.orchestration_parameters.perspective_count assert ( - len(list(filter(lambda perspective: perspective.check_type == CheckType.CAA, perspectives_list))) + len(list(filter(lambda perspective: perspective.check_response.check_type == CheckType.CAA, perspectives_list))) == request.orchestration_parameters.perspective_count ) @@ -276,10 +276,8 @@ def api_should_return_200_given_valid_website_change_validation( request = MpicDcvRequest( domain_or_ip_target=domain_or_ip_target, orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), - dcv_check_parameters=DcvCheckParameters( - validation_details=DcvWebsiteChangeValidationParameters( - http_token_path=http_token_path, challenge_value=challenge_value - ) + dcv_check_parameters=DcvWebsiteChangeValidationParameters( + http_token_path=http_token_path, challenge_value=challenge_value ), ) print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body From 8bdb3aa3ed09719bec46ffd402706633f60bea5f Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 4 Feb 2025 20:35:04 -0500 Subject: [PATCH 07/10] added api spec version to toml --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 322daa6..2f10e14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,10 @@ Source = "https://github.com/open-mpic/aws-lambda-python" #[dirs.env] #virtual = ".hatch" +[tool.api] +spec_version = "3.0.0" +spec_repository = "https://github.com/open-mpic/open-mpic-specification" + [tool.hatch] version.path = "src/aws_lambda_mpic/__about__.py" build.sources = ["src", "resources"] From def2ebd9cae15c33910cb800eb0b6b5512dd9c18 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Wed, 5 Feb 2025 15:50:04 -0500 Subject: [PATCH 08/10] updated build process slightly to be able to call everything from hatch, fine grained and coarse grained. renamed lambda python layer to be just python3_ instead of python311_ to make it less of a hassle to later to go to 3.13 and such --- README.md | 14 ++++++++------ clean.sh | 5 +---- deploy.sh | 2 +- layer/package.sh | 19 ------------------- open-tofu/aws-perspective.tf.template | 12 ++++++------ open-tofu/main.tf.template | 10 +++++----- open-tofu/variables.tf | 2 +- package-layer.sh | 17 +++++++++++++++++ pyproject.toml | 25 +++++++++++++++++++++---- 9 files changed, 60 insertions(+), 46 deletions(-) delete mode 100755 layer/package.sh create mode 100755 package-layer.sh diff --git a/README.md b/README.md index dffd2db..954098e 100644 --- a/README.md +++ b/README.md @@ -13,16 +13,18 @@ All requirements for running the API are packaged and uploaded to AWS as a lambd - Hatch (https://hatch.pypa.io/) for building and running the project. This is a Python project manager that can be installed via `pip install hatch`. ## Deployment Steps -1. From the project root directory, run `hatch run lambda-layer:install`. This will create a virtual Python environment in the `layer` directory and install the project dependencies via pip. -2. Package the AWS layer. In the `layer` directory, run `./package.sh`. This will make two files: `python311_layer_content.zip` and `mpic_coordinator_layer_content.zip` which will later be referenced by Open Tofu. -3. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` -4. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. -5. Run `hatch run ./configure.py` from the root directory of the repo to generate Open Tofu files from templates. +1. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. +2. Create a virtual Python environment in the `layer` directory and install the project dependencies via pip. This can be executed by running `hatch run lambda:layer-install`. +3. Package two AWS layers by executing `package-layer.sh`. This will make two files: `python3_layer_content.zip` and `mpic_coordinator_layer_content.zip` which will later be referenced by Open Tofu. This can be done by running `./package-layer.sh` or `hatch run lambda:layer-package`. +4. Run `configure.py` from the root directory of the repo to generate Open Tofu files from templates. This can be separately executed by running `hatch run ./configure.py` or `hatch run lambda:configure-tf`. +5. Zip all Lambda functions. AWS Lambda functions are usually deployed from zip files. This can be separately executed by running `./zip-all.sh` or `hatch run lambda:zip-all`. 6. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. This provides a standard install with DNSSEC enabled which causes the system to incur expenses even when it is not in use (due to the AWS VPC NAT Gateways needed). To reduce the AWS bill, DNSSEC can also be disabled by appending `-var="dnssec_enabled=false"` to `tofu apply` (i.e., `tofu apply -var="dnssec_enabled=false"`). 7. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. 8. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. -For convenience `./deploy.sh` in the project root will perform all of these steps (using `-var="dnssec_enabled=false"`) with the exception of copying over the example config to the operational config and running `tofu init` in the open-tofu dir. +For convenience `./deploy.sh` in the project root will clean the environment and perform all of these steps (using `-var="dnssec_enabled=false"`), with the exception of copying over the example config to the operational config and running `tofu init` in the open-tofu dir. +In addition, `hatch run lambda:prepare` will run steps 2-5 in a single command. +Finally, `hatch run lambda:deploy` will clean the environment and then run steps 2-6, in the same manner as `deploy.sh`. ## Testing The following is an example of a test API call that uses bash command substitution to fill in the proper values for the API URL and the API key. diff --git a/clean.sh b/clean.sh index fe79891..8f2c045 100755 --- a/clean.sh +++ b/clean.sh @@ -7,11 +7,8 @@ FUNCTIONS_DIR="src/aws_lambda_mpic" rm open-tofu/*.generated.tf -rm -r layer/create_layer_virtualenv -rm -r layer/python311_layer_content -rm -r layer/mpic_coordinator_layer_content - rm layer/*.zip +rm -r layer/create_layer_virtualenv rm "${FUNCTIONS_DIR}"/mpic_coordinator_lambda/mpic_coordinator_lambda.zip rm "${FUNCTIONS_DIR}"/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip diff --git a/deploy.sh b/deploy.sh index 1d73363..878954c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,2 +1,2 @@ #!/bin/bash -./clean.sh; hatch env prune; hatch run lambda-layer:install; cd layer; ./package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -var="dnssec_enabled=false" -auto-approve; cd .. +./clean.sh; hatch env prune; hatch run lambda-layer:install; ./package-layer.sh; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -var="dnssec_enabled=false" -auto-approve; cd .. diff --git a/layer/package.sh b/layer/package.sh deleted file mode 100755 index c9841f1..0000000 --- a/layer/package.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# make common python3.11 layer for all lambda functions -mkdir -p python311_layer_content/python -cd python311_layer_content -cp -r ../create_layer_virtualenv/lib python/ -zip -r ../python311_layer_content.zip python -cd .. # should be at layer directory - -py_exclude=('*.pyc' '*__pycache__*') - -# make mpic_coordinator lambda layer for mpic coordinator lambda function -mkdir -p mpic_coordinator_layer_content/python -cp -r ../resources mpic_coordinator_layer_content/python/resources # TODO consider a more elegant approach -cd mpic_coordinator_layer_content -zip -r ../mpic_coordinator_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_coordinator lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - - diff --git a/open-tofu/aws-perspective.tf.template b/open-tofu/aws-perspective.tf.template index 88f9812..f2665e1 100644 --- a/open-tofu/aws-perspective.tf.template +++ b/open-tofu/aws-perspective.tf.template @@ -1,8 +1,8 @@ # Each layer must be created in the region of the functions. -resource "aws_lambda_layer_version" "python311_open_mpic_layer_{{region}}" { - filename = "../layer/python311_layer_content.zip" - layer_name = "python311_open_mpic_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/python311_layer_content.zip")}" +resource "aws_lambda_layer_version" "python3_open_mpic_layer_{{region}}" { + filename = "../layer/python3_layer_content.zip" + layer_name = "python3_open_mpic_layer_{{region}}_{{deployment-id}}" + source_code_hash = "${filebase64sha256("../layer/python3_layer_content.zip")}" compatible_runtimes = ["python3.11"] provider = aws.{{region}} } @@ -211,7 +211,7 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { runtime = "python3.11" architectures = ["arm64"] layers = [ - aws_lambda_layer_version.python311_open_mpic_layer_{{region}}.arn, + aws_lambda_layer_version.python3_open_mpic_layer_{{region}}.arn, ] vpc_config { subnet_ids = [for s in aws_subnet.subnet_private_{{region}} : s.id] @@ -241,7 +241,7 @@ resource "aws_lambda_function" "mpic_caa_checker_lambda_{{region}}" { runtime = "python3.11" architectures = ["arm64"] layers = [ - aws_lambda_layer_version.python311_open_mpic_layer_{{region}}.arn, + aws_lambda_layer_version.python3_open_mpic_layer_{{region}}.arn, ] vpc_config { subnet_ids = [for s in aws_subnet.subnet_private_{{region}} : s.id] diff --git a/open-tofu/main.tf.template b/open-tofu/main.tf.template index 04f406e..dc694c2 100644 --- a/open-tofu/main.tf.template +++ b/open-tofu/main.tf.template @@ -5,10 +5,10 @@ provider "aws" { } # Python open-mpic layer (contains third party libraries) -resource "aws_lambda_layer_version" "python311_open_mpic_layer" { - filename = "../layer/python311_layer_content.zip" - layer_name = "python_311_open_mpic_layer_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/python311_layer_content.zip")}" +resource "aws_lambda_layer_version" "python3_open_mpic_layer" { + filename = "../layer/python3_layer_content.zip" + layer_name = "python3_open_mpic_layer_{{deployment-id}}" + source_code_hash = "${filebase64sha256("../layer/python3_layer_content.zip")}" compatible_runtimes = ["python3.11"] } @@ -75,7 +75,7 @@ resource "aws_lambda_function" "mpic_coordinator_lambda" { architectures = ["arm64"] timeout = 60 layers = [ - aws_lambda_layer_version.python311_open_mpic_layer.arn, + aws_lambda_layer_version.python3_open_mpic_layer.arn, aws_lambda_layer_version.mpic_coordinator_layer.arn, ] environment { diff --git a/open-tofu/variables.tf b/open-tofu/variables.tf index 05a1c85..b359366 100644 --- a/open-tofu/variables.tf +++ b/open-tofu/variables.tf @@ -1,5 +1,5 @@ variable "dnssec_enabled" { type = bool description = "Enable DNSSEC" - default = true + default = false } diff --git a/package-layer.sh b/package-layer.sh new file mode 100755 index 0000000..bb39dbe --- /dev/null +++ b/package-layer.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# make common python3 layer for all lambda functions +mkdir -p layer/python3_layer_content/python +cp -r layer/create_layer_virtualenv/lib layer/python3_layer_content/python/ +(cd layer/python3_layer_content && zip -r ../python3_layer_content.zip python) + +py_exclude=("*.pyc" "*__pycache__*" "*.pyo" "*.pyd") + +# make mpic_coordinator lambda layer for mpic coordinator lambda function +mkdir -p layer/mpic_coordinator_layer_content/python +cp -r resources layer/mpic_coordinator_layer_content/python/resources # TODO consider a more elegant approach +# Zip the mpic_coordinator lambda layer +(cd layer/mpic_coordinator_layer_content && zip -r ../mpic_coordinator_layer_content.zip python -x "${py_exclude[@]}") + +# clean up, mostly for the IDE which could otherwise detect duplicate code +rm -r layer/python3_layer_content +rm -r layer/mpic_coordinator_layer_content diff --git a/pyproject.toml b/pyproject.toml index 2f10e14..a85d8d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,20 +74,37 @@ PIP_INDEX_URL = "https://pypi.org/simple/" #PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" # FIXME here temporarily to test open-mpic-core packaging PIP_VERBOSE = "1" -[tool.hatch.envs.lambda-layer] +[tool.hatch.envs.lambda] skip-install = true python = "3.11" type="virtual" path="layer/create_layer_virtualenv" -[tool.hatch.envs.lambda-layer.env-vars] +[tool.hatch.envs.lambda.env-vars] #PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" PIP_ONLY_BINARY = ":all:" #PIP_PLATFORM = "manylinux2014_aarch64" #PIP_TARGET = "layer/create_layer_virtualenv2/lib/python3.11/site-packages" # does not work... bug in pip 24.2? -[tool.hatch.envs.lambda-layer.scripts] -install = "pip install . --platform manylinux2014_aarch64 --only-binary=:all: --target layer/create_layer_virtualenv/lib/python3.11/site-packages" +[tool.hatch.envs.lambda.scripts] +layer-install = "pip install . --platform manylinux2014_aarch64 --only-binary=:all: --target layer/create_layer_virtualenv/lib/python3.11/site-packages" +layer-package = "sh ./package-layer.sh" +configure-tf = "python configure.py" +zip-lambdas = "sh ./zip-all.sh" +apply-tf = "(cd open-tofu && tofu apply -var=\"dnssec_enabled=false\" -auto-approve)" +prepare = [ + "layer-install", + "layer-package", + "configure-tf", + "zip-lambdas" +] +clean = "sh ./clean.sh" +deploy = [ + "clean", + "prepare", + "apply-tf" +] + [tool.hatch.envs.test] skip-install = false From 8103664637be54a2db690452c941d85925b0e8dd Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Wed, 5 Feb 2025 15:53:27 -0500 Subject: [PATCH 09/10] tweaked readme --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 954098e..4353eac 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,10 @@ All requirements for running the API are packaged and uploaded to AWS as a lambd 7. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. 8. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. -For convenience `./deploy.sh` in the project root will clean the environment and perform all of these steps (using `-var="dnssec_enabled=false"`), with the exception of copying over the example config to the operational config and running `tofu init` in the open-tofu dir. -In addition, `hatch run lambda:prepare` will run steps 2-5 in a single command. -Finally, `hatch run lambda:deploy` will clean the environment and then run steps 2-6, in the same manner as `deploy.sh`. +For convenience: +* `./deploy.sh` in the project root will clean the environment and perform steps 2-6 (using `-var="dnssec_enabled=false"`), with the exception of copying over the example config to the operational config and running `tofu init` in the open-tofu dir. +* `hatch run lambda:prepare` will run steps 2-5 in a single command. +* `hatch run lambda:deploy` will clean the environment and then run steps 2-6, in the same manner as `deploy.sh`. ## Testing The following is an example of a test API call that uses bash command substitution to fill in the proper values for the API URL and the API key. From f571f5c6f0846ba2e29630c1c818b0fd2f1121b2 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Thu, 6 Feb 2025 12:26:57 -0500 Subject: [PATCH 10/10] pointing to v5 of published core --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a85d8d6..73db6be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,14 +26,14 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@birgelee-dcv-caa-perspective-code-response", +# "open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@birgelee-dcv-caa-perspective-code-response", "pyyaml==6.0.1", "requests>=2.32.3", "dnspython==2.6.1", "pydantic==2.8.2", "aiohttp==3.11.11", "aws-lambda-powertools[parser]==3.2.0", - #"open-mpic-core==4.7.2", + "open-mpic-core==5.0.0", "aioboto3~=13.3.0", "black==24.8.0", ]