diff --git a/pyproject.toml b/pyproject.toml index 1aaff2f..e7fdee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,14 +26,14 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ -# "open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@main", + #"open-mpic-core @ git+https://github.com/open-mpic/open-mpic-core-python.git@ds-trace-logging", "pyyaml==6.0.1", "requests>=2.32.3", "dnspython==2.6.1", "pydantic==2.8.2", "aiohttp==3.11.11", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==4.4.0", + "open-mpic-core==4.5.0", "aioboto3~=13.3.0", ] diff --git a/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py b/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py index 25cb41a..87df51d 100644 --- a/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_caa_checker_lambda/mpic_caa_checker_lambda_function.py @@ -1,17 +1,28 @@ +import os import asyncio from aws_lambda_powertools.utilities.parser import event_parser from open_mpic_core.common_domain.check_request import CaaCheckRequest from open_mpic_core.mpic_caa_checker.mpic_caa_checker import MpicCaaChecker -import os +from open_mpic_core.common_util.trace_level_logger import get_logger + +logger = get_logger(__name__) class MpicCaaCheckerLambdaHandler: def __init__(self): self.perspective_code = os.environ['AWS_REGION'] self.default_caa_domain_list = os.environ['default_caa_domains'].split("|") - self.caa_checker = MpicCaaChecker(self.default_caa_domain_list, self.perspective_code) + self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None + + self.logger = logger.getChild(self.__class__.__name__) + if self.log_level: + self.logger.setLevel(self.log_level) + + self.caa_checker = MpicCaaChecker(default_caa_domain_list=self.default_caa_domain_list, + perspective_code=self.perspective_code, + log_level=self.logger.level) def process_invocation(self, caa_request: CaaCheckRequest): try: diff --git a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py index fba8820..a6d905e 100644 --- a/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_coordinator_lambda/mpic_coordinator_lambda_function.py @@ -1,5 +1,8 @@ +import logging import os import json +import traceback + import yaml import asyncio import aioboto3 @@ -7,9 +10,9 @@ from asyncio import Queue from collections import defaultdict from importlib import resources - -from aws_lambda_powertools.utilities.parser import event_parser, envelopes from pydantic import TypeAdapter, ValidationError, BaseModel +from aws_lambda_powertools.utilities.parser import event_parser, envelopes + from open_mpic_core.common_domain.check_request import BaseCheckRequest from open_mpic_core.common_domain.check_response import CheckResponse from open_mpic_core.mpic_coordinator.domain.mpic_request import MpicRequest @@ -18,6 +21,9 @@ from open_mpic_core.mpic_coordinator.mpic_coordinator import MpicCoordinator, MpicCoordinatorConfiguration from open_mpic_core.common_domain.enum.check_type import CheckType from open_mpic_core.mpic_coordinator.domain.remote_perspective import RemotePerspective +from open_mpic_core.common_util.trace_level_logger import get_logger + +logger = get_logger(__name__) class PerspectiveEndpointInfo(BaseModel): @@ -37,6 +43,11 @@ def __init__(self): self.default_perspective_count = int(os.environ['default_perspective_count']) self.global_max_attempts = int(os.environ['absolute_max_attempts']) if 'absolute_max_attempts' in os.environ else None self.hash_secret = os.environ['hash_secret'] + self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None + + self.logger = logger.getChild(self.__class__.__name__) + if self.log_level: + self.logger.setLevel(self.log_level) self.remotes_per_perspective_per_check_type = { CheckType.DCV: {perspective_code: perspective_config.dcv_endpoint_info for perspective_code, perspective_config in perspectives.items()}, @@ -54,10 +65,7 @@ def __init__(self): self.hash_secret ) - self.mpic_coordinator = MpicCoordinator( - self.call_remote_perspective, - self.mpic_coordinator_configuration - ) + self.mpic_coordinator = MpicCoordinator(self.call_remote_perspective, self.mpic_coordinator_configuration, self.logger.level) # for correct deserialization of responses based on discriminator field (check type) self.mpic_request_adapter = TypeAdapter(MpicRequest) @@ -125,7 +133,7 @@ async def call_remote_perspective(self, perspective: RemotePerspective, check_ty response_payload = json.loads(await response['Payload'].read()) return self.check_response_adapter.validate_json(response_payload['body']) except ValidationError as ve: - # We might want to handle this differently later. + self.logger.log(level=logging.ERROR, msg=f"Validation error in response from {perspective.code}: {ve}") raise ve finally: await self.release_lambda_client(perspective.code, client) @@ -184,6 +192,9 @@ def wrapper(*args, **kwargs): except ValidationError as validation_error: return build_400_response(MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key, validation_error.errors()) except Exception as e: + logger.error(f"An error occurred: {str(e)}") + print(traceback.format_exc()) + print(f"BOY HOWDY error occurred: {str(e)}") return { 'statusCode': 500, 'headers': {'Content-Type': 'application/json'}, diff --git a/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py b/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py index 7b9c357..8a4bb4e 100644 --- a/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py +++ b/src/aws_lambda_mpic/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda_function.py @@ -1,16 +1,25 @@ +import os import asyncio from aws_lambda_powertools.utilities.parser import event_parser from open_mpic_core.common_domain.check_request import DcvCheckRequest from open_mpic_core.mpic_dcv_checker.mpic_dcv_checker import MpicDcvChecker -import os +from open_mpic_core.common_util.trace_level_logger import get_logger + +logger = get_logger(__name__) class MpicDcvCheckerLambdaHandler: def __init__(self): self.perspective_code = os.environ['AWS_REGION'] - self.dcv_checker = MpicDcvChecker(self.perspective_code) + self.log_level = os.environ['log_level'] if 'log_level' in os.environ else None + + self.logger = logger.getChild(self.__class__.__name__) + if self.log_level: + self.logger.setLevel(self.log_level) + + self.dcv_checker = MpicDcvChecker(perspective_code=self.perspective_code, log_level=self.logger.level) def process_invocation(self, dcv_request: DcvCheckRequest): try: diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index 1050753..5c3d4fc 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -1,6 +1,7 @@ import json import sys import pytest + from pydantic import TypeAdapter from open_mpic_core.common_domain.check_parameters import CaaCheckParameters, DcvWebsiteChangeValidationDetails diff --git a/tests/unit/aws_lambda_mpic/conftest.py b/tests/unit/aws_lambda_mpic/conftest.py new file mode 100644 index 0000000..a113734 --- /dev/null +++ b/tests/unit/aws_lambda_mpic/conftest.py @@ -0,0 +1,21 @@ +# conftest.py +import logging +from io import StringIO +import pytest + + +@pytest.fixture(autouse=True) +def setup_logging(): + # Clear existing handlers + root = logging.getLogger() + for handler in root.handlers[:]: + root.removeHandler(handler) + + log_output = StringIO() # to be able to inspect what gets logged + handler = logging.StreamHandler(log_output) + handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) + + # Configure fresh logging + logging.basicConfig(handlers=[handler]) + + yield log_output diff --git a/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py index 1c44a3b..9fdcc55 100644 --- a/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_caa_checker_lambda.py @@ -1,19 +1,23 @@ import time +import dns import pytest import aws_lambda_mpic.mpic_caa_checker_lambda.mpic_caa_checker_lambda_function as mpic_caa_checker_lambda_function from open_mpic_core.common_domain.check_response import CaaCheckResponse, CaaCheckResponseDetails +from open_mpic_core_test.test_util.mock_dns_object_creator import MockDnsObjectCreator from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator +# noinspection PyMethodMayBeStatic class TestCaaCheckerLambda: @staticmethod @pytest.fixture(scope='class') def set_env_variables(): envvars = { 'AWS_REGION': 'us-east-1', - 'default_caa_domains': 'ca1.com|ca2.org|ca3.net' + 'default_caa_domains': 'ca1.com|ca2.org|ca3.net', + 'log_level': 'TRACE' } with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: for k, v in envvars.items(): @@ -33,6 +37,21 @@ def lambda_handler__should_do_caa_check_using_configured_caa_checker(self, set_e result = mpic_caa_checker_lambda_function.lambda_handler(caa_check_request, None) assert result == mock_return_value + def lambda_handler__should_set_log_level_of_caa_checker(self, set_env_variables, setup_logging, mocker): + caa_check_request = ValidCheckCreator.create_valid_caa_check_request() + + records = [MockDnsObjectCreator.create_caa_record(0, 'issue', 'ca1.org')] + mock_rrset = MockDnsObjectCreator.create_rrset(dns.rdatatype.CAA, *records) + mock_domain = dns.name.from_text(caa_check_request.domain_or_ip_target) + mock_return = (mock_rrset, mock_domain) + mocker.patch('open_mpic_core.mpic_caa_checker.mpic_caa_checker.MpicCaaChecker.find_caa_records_and_domain', + return_value=mock_return) + + result = mpic_caa_checker_lambda_function.lambda_handler(caa_check_request, None) + assert result['statusCode'] == 200 + log_contents = setup_logging.getvalue() + assert all(text in log_contents for text in ['MpicCaaChecker', 'TRACE']) # Verify the log level was set + @staticmethod def create_caa_check_response(): return CaaCheckResponse(perspective_code='us-east-1', check_passed=True, diff --git a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py index fa8e944..ad70319 100644 --- a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py @@ -1,4 +1,5 @@ import time + import pytest import aws_lambda_mpic.mpic_dcv_checker_lambda.mpic_dcv_checker_lambda_function as mpic_dcv_checker_lambda_function @@ -8,13 +9,17 @@ from open_mpic_core.common_domain.check_response import DcvCheckResponse from open_mpic_core_test.test_util.valid_check_creator import ValidCheckCreator +from unit.aws_lambda_mpic.conftest import setup_logging + +# noinspection PyMethodMayBeStatic class TestDcvCheckerLambda: @staticmethod @pytest.fixture(scope='class') def set_env_variables(): envvars = { 'AWS_REGION': 'us-east-1', + 'log_level': 'TRACE' } with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: for k, v in envvars.items(): @@ -53,6 +58,15 @@ def lambda_handler__should_return_appropriate_status_code_given_errors_in_respon result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None) assert result == mock_return_value + def lambda_handler__should_set_log_level_of_dcv_checker(self, set_env_variables, mocker, setup_logging): + dcv_check_request = ValidCheckCreator.create_valid_http_check_request() + mocker.patch('open_mpic_core.mpic_dcv_checker.mpic_dcv_checker.MpicDcvChecker.perform_http_based_validation', + return_value=TestDcvCheckerLambda.create_dcv_check_response()) + result = mpic_dcv_checker_lambda_function.lambda_handler(dcv_check_request, None) + assert result['statusCode'] == 200 + log_contents = setup_logging.getvalue() + assert all(text in log_contents for text in ['MpicDcvChecker', 'TRACE']) # Verify the log level was set + @staticmethod def create_dcv_check_response(): return DcvCheckResponse(perspective_code='us-east-1', check_passed=True, diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index ba726fd..46e888f 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -12,9 +12,9 @@ from pydantic import TypeAdapter from open_mpic_core.common_domain.check_request import DcvCheckRequest -from open_mpic_core.common_domain.check_response import DcvCheckResponse +from open_mpic_core.common_domain.check_response import DcvCheckResponse, CaaCheckResponse from open_mpic_core.common_domain.enum.check_type import CheckType -from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails +from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails, CaaCheckResponseDetails from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod from open_mpic_core.mpic_coordinator.domain.mpic_orchestration_parameters import MpicEffectiveOrchestrationParameters from open_mpic_core.mpic_coordinator.domain.mpic_response import MpicCaaResponse @@ -49,7 +49,8 @@ def set_env_variables(): envvars = { 'perspectives': json.dumps({k: v.model_dump() for k, v in perspectives_as_dict.items()}), 'default_perspective_count': '3', - 'hash_secret': 'test_secret' + 'hash_secret': 'test_secret', + 'log_level': 'TRACE' } with pytest.MonkeyPatch.context() as class_scoped_monkeypatch: for k, v in envvars.items(): @@ -153,6 +154,28 @@ def lambda_handler__should_coordinate_mpic_using_configured_mpic_coordinator(sel result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) assert result == expected_response + def lambda_handler__should_set_log_level_for_coordinator(self, set_env_variables, setup_logging, mocker): + mpic_request = ValidMpicRequestCreator.create_valid_mpic_request(CheckType.CAA) + api_request = TestMpicCoordinatorLambda.create_api_gateway_request() + api_request.body = mpic_request.model_dump_json() + mocked_perspective_responses = [ + CaaCheckResponse(perspective_code='us-east-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponse(perspective_code='us-west-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponse(perspective_code='eu-west-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponse(perspective_code='eu-central-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponse(perspective_code='ap-northeast-1', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + CaaCheckResponse(perspective_code='ap-south-2', check_passed=True, details=CaaCheckResponseDetails(caa_record_present=False)), + ] + mocked_validity_per_perspective = {response.perspective_code: response.check_passed for response in mocked_perspective_responses} + mock_return = (mocked_perspective_responses, mocked_validity_per_perspective) + + mocker.patch('open_mpic_core.mpic_coordinator.mpic_coordinator.MpicCoordinator.issue_async_calls_and_collect_responses', return_value=mock_return) + # noinspection PyTypeChecker + result = mpic_coordinator_lambda_function.lambda_handler(api_request, None) + assert result['statusCode'] == 200 + log_contents = setup_logging.getvalue() + assert all(text in log_contents for text in ['MpicCoordinator', 'TRACE']) # Verify the log level was set + def load_aws_region_config__should_return_dict_of_aws_regions_with_proximity_info_by_region_code(self): mpic_coordinator_lambda_handler = MpicCoordinatorLambdaHandler() loaded_aws_regions = mpic_coordinator_lambda_handler.load_aws_region_config() @@ -213,6 +236,14 @@ def create_caa_mpic_response(): caa_check_parameters=caa_request.caa_check_parameters ) + @staticmethod + def create_caa_perspective_response(*args, **kwargs) -> CaaCheckResponse: + return CaaCheckResponse( + perspective_code=kwargs['perspective'].code, + check_passed=True, + details=CaaCheckResponseDetails(caa_record_present=False), + ) + @staticmethod def create_api_gateway_request(): request = APIGatewayProxyEventModel(