From 28afab594b98ca8ef35468cc256560a8dde53f8a Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Mon, 9 Dec 2024 21:27:40 -0700 Subject: [PATCH 1/9] tweaked project to use pypi package instead of git submodule. updated to use open-mpic-core v2.2.0 --- .gitmodules | 3 --- open_mpic_core_python | 1 - pyproject.toml | 8 +++++++- tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py | 8 ++++---- .../unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py | 5 +++-- 5 files changed, 14 insertions(+), 11 deletions(-) delete mode 100644 .gitmodules delete mode 160000 open_mpic_core_python diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a7275d1..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "open_mpic_core_python"] - path = open_mpic_core_python - url = https://github.com/open-mpic/open-mpic-core-python.git diff --git a/open_mpic_core_python b/open_mpic_core_python deleted file mode 160000 index 4f3824a..0000000 --- a/open_mpic_core_python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4f3824a450e452f18796d8fd55c3be934b0ae498 diff --git a/pyproject.toml b/pyproject.toml index 0ad477c..48571f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "dnspython==2.6.1", "pydantic==2.8.2", "aws-lambda-powertools[parser]==3.2.0", + "open-mpic-core==2.2.0" ] [project.optional-dependencies] @@ -60,6 +61,11 @@ skip-install = false type="virtual" path="venv" +[tool.hatch.envs.default.env-vars] +PIP_INDEX_URL = "https://pypi.org/simple/" +PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" # FIXME here temporarily to test open-mpic-core packaging +PIP_VERBOSE = "1" + [tool.hatch.envs.test] features = [ "test", @@ -70,7 +76,7 @@ features = [ unit = "pytest" unit-html = "pytest --html=testreports/index.html" # generate html report (warning: uses an aging plugin, 11-2023) integration = "pytest tests/integration" -coverage = "pytest --cov=src/aws_lambda_mpic --cov=src/open_mpic_core --cov-report=term-missing --cov-report=html" +coverage = "pytest --cov=src/aws_lambda_mpic --cov-report=term-missing --cov-report=html" [tool.hatch.envs.hatch-test] features = [ diff --git a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py index 8e54815..1709681 100644 --- a/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_dcv_checker_lambda.py @@ -2,11 +2,11 @@ import pytest import aws_lambda_mpic.mpic_dcv_checker_lambda.mpic_dcv_checker_lambda_function as mpic_dcv_checker_lambda_function -from open_mpic_core.common_domain.check_parameters import DcvWebsiteChangeValidationDetails -from open_mpic_core.common_domain.check_response import DcvCheckResponse, DcvCheckResponseDetails -from open_mpic_core.common_domain.check_response_details import DcvWebsiteChangeResponseDetails from open_mpic_core.common_domain.validation_error import MpicValidationError from unit.test_util.valid_check_creator import ValidCheckCreator +from open_mpic_core.common_domain.check_response_details import DcvHttpCheckResponseDetails +from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod +from open_mpic_core.common_domain.check_response import DcvCheckResponse class TestDcvCheckerLambda: @@ -58,7 +58,7 @@ def lambda_handler__should_return_appropriate_status_code_given_errors_in_respon @staticmethod def create_dcv_check_response(): return DcvCheckResponse(perspective_code='us-east-1', check_passed=True, - details=DcvWebsiteChangeResponseDetails(), + details=DcvHttpCheckResponseDetails(validation_method=DcvValidationMethod.WEBSITE_CHANGE_V2), timestamp_ns=time.time_ns()) diff --git a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py index c887a33..e7bf98d 100644 --- a/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py +++ b/tests/unit/aws_lambda_mpic/test_mpic_coordinator_lambda.py @@ -12,9 +12,10 @@ from open_mpic_core.common_domain.check_request import DcvCheckRequest from open_mpic_core.common_domain.check_response import DcvCheckResponse -from open_mpic_core.common_domain.check_response_details import DcvDnsChangeResponseDetails from open_mpic_core.common_domain.enum.check_type import CheckType from open_mpic_core.common_domain.remote_perspective import RemotePerspective +from open_mpic_core.common_domain.check_response_details import DcvDnsCheckResponseDetails +from open_mpic_core.common_domain.enum.dcv_validation_method import DcvValidationMethod from open_mpic_core.mpic_coordinator.domain.mpic_orchestration_parameters import MpicEffectiveOrchestrationParameters from open_mpic_core.mpic_coordinator.domain.mpic_response import MpicCaaResponse from aws_lambda_mpic.mpic_coordinator_lambda.mpic_coordinator_lambda_function import MpicCoordinatorLambdaHandler @@ -122,7 +123,7 @@ def create_successful_boto3_api_call_response_for_dcv_check(self, lambda_method, check_request = DcvCheckRequest.model_validate_json(lambda_configuration['Payload']) # hijacking the value of 'perspective' to verify that the right arguments got passed to the call expected_response_body = DcvCheckResponse(perspective_code=check_request.domain_or_ip_target, - check_passed=True, details=DcvDnsChangeResponseDetails()) + check_passed=True, details=DcvDnsCheckResponseDetails(validation_method=DcvValidationMethod.ACME_DNS_01)) expected_response = {'statusCode': 200, 'body': expected_response_body.model_dump_json()} json_bytes = json.dumps(expected_response).encode('utf-8') file_like_response = io.BytesIO(json_bytes) From e570171f2bcae80ea3fd79cfaf1ff1dbd8a9d9f2 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 10 Dec 2024 16:54:31 -0700 Subject: [PATCH 2/9] making layer 1-package obsolete so we can remove requirements.txt --- clean.sh | 6 -- configure.py | 9 +- layer/2-package.sh | 26 +---- ...quirements.txt => requirements.txt.backup} | 0 open-tofu/aws-perspective.tf.template | 101 ++++++++---------- open-tofu/main.tf.template | 10 +- open-tofu/variables.tf | 5 + pyproject.toml | 48 +++++++-- tests/integration/test_deployed_mpic_api.py | 50 ++++----- 9 files changed, 123 insertions(+), 132 deletions(-) rename layer/{requirements.txt => requirements.txt.backup} (100%) create mode 100644 open-tofu/variables.tf diff --git a/clean.sh b/clean.sh index a3c871a..cf0d5c1 100755 --- a/clean.sh +++ b/clean.sh @@ -10,15 +10,9 @@ rm open-tofu/*.generated.tf rm -r layer/create_layer_virtualenv rm -r layer/python311_layer_content rm -r layer/mpic_coordinator_layer_content -rm -r layer/mpic_caa_checker_layer_content -rm -r layer/mpic_dcv_checker_layer_content -rm -r layer/mpic_common_layer_content rm layer/python311_layer_content.zip rm layer/mpic_coordinator_layer_content.zip -rm layer/mpic_caa_checker_layer_content.zip -rm layer/mpic_dcv_checker_layer_content.zip -rm layer/mpic_common_layer_content.zip rm "${FUNCTIONS_DIR}"/mpic_coordinator_lambda/mpic_coordinator_lambda.zip rm "${FUNCTIONS_DIR}"/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip diff --git a/configure.py b/configure.py index 9430f1c..a3be590 100755 --- a/configure.py +++ b/configure.py @@ -38,7 +38,6 @@ def main(raw_args=None): stream.write(deployment_id_to_write) # Read the deployment id. - deployment_id = 0 with open(args.deployment_id_file) as stream: deployment_id = int(stream.read()) @@ -48,14 +47,14 @@ def main(raw_args=None): try: config = yaml.safe_load(stream) except yaml.YAMLError as exc: - print(f"Error loading YAML config at {args.config}. Project not configured. Error details: {exec}.") + print(f"Error loading YAML config at {args.config}. Project not configured. Error details: {exc}.") exit() aws_available_regions = {} with open(args.available_regions) as stream: try: aws_available_regions = yaml.safe_load(stream)['aws-available-regions'] except yaml.YAMLError as exc: - print(f"Error loading YAML config at {args.available_regions}. Project not configured. Error details: {exec}.") + print(f"Error loading YAML config at {args.available_regions}. Project not configured. Error details: {exc}.") exit() # Remove all old files. @@ -96,7 +95,6 @@ def main(raw_args=None): main_tf_string = main_tf_string.replace("{{absolut-max-attempts-with-key}}", f"absolute_max_attempts = \"{config['absolute-max-attempts']}\"") else: main_tf_string = main_tf_string.replace("{{absolut-max-attempts-with-key}}", "") - # Replace enforce distinct rir regions. main_tf_string = main_tf_string.replace("{{enforce-distinct-rir-regions}}", f"\"{1 if config['enforce-distinct-rir-regions'] else 0}\"") @@ -154,7 +152,6 @@ def main(raw_args=None): # Set the RIR region to load into env variables. aws_perspective_tf_region = aws_perspective_tf_region.replace("{{rir-region}}", f"{rir_region}") - if not args.aws_perspective_tf_template.endswith(".tf.template"): print(f"Error: invalid tf template name: {args.aws_perspective_tf_template}. Make sure all tf template files end in '.tf.template'.") exit() @@ -166,4 +163,4 @@ def main(raw_args=None): # Main module init for direct invocation. if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/layer/2-package.sh b/layer/2-package.sh index ae56758..c9841f1 100755 --- a/layer/2-package.sh +++ b/layer/2-package.sh @@ -8,36 +8,12 @@ cd .. # should be at layer directory py_exclude=('*.pyc' '*__pycache__*') -# make mpic_common lambda layer for all lambda functions -mkdir -p mpic_common_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/common_domain mpic_common_layer_content/python/open_mpic_core/common_domain -cd mpic_common_layer_content -zip -r ../mpic_common_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_common lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - # make mpic_coordinator lambda layer for mpic coordinator lambda function -mkdir -p mpic_coordinator_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/mpic_coordinator mpic_coordinator_layer_content/python/open_mpic_core/mpic_coordinator +mkdir -p mpic_coordinator_layer_content/python cp -r ../resources mpic_coordinator_layer_content/python/resources # TODO consider a more elegant approach cd mpic_coordinator_layer_content zip -r ../mpic_coordinator_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_coordinator lambda layer rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! cd .. # should be at layer directory -# make mpic_caa_checker lambda layer for mpic caa checker lambda function -mkdir -p mpic_caa_checker_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/mpic_caa_checker mpic_caa_checker_layer_content/python/open_mpic_core/mpic_caa_checker -cd mpic_caa_checker_layer_content -zip -r ../mpic_caa_checker_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_caa_checker lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory - -# make mpic_dcv_checker lambda layer for mpic dcv checker lambda function -mkdir -p mpic_dcv_checker_layer_content/python/open_mpic_core -cp -r ../open_mpic_core_python/src/open_mpic_core/mpic_dcv_checker mpic_dcv_checker_layer_content/python/open_mpic_core/mpic_dcv_checker -cd mpic_dcv_checker_layer_content -zip -r ../mpic_dcv_checker_layer_content.zip python -x "${py_exclude[@]}" # Zip the mpic_dcv_checker lambda layer -rm -r python # clean up, mostly not to bother the IDE which will find this duplicate code! -cd .. # should be at layer directory diff --git a/layer/requirements.txt b/layer/requirements.txt.backup similarity index 100% rename from layer/requirements.txt rename to layer/requirements.txt.backup diff --git a/open-tofu/aws-perspective.tf.template b/open-tofu/aws-perspective.tf.template index 904fada..3c3b3db 100644 --- a/open-tofu/aws-perspective.tf.template +++ b/open-tofu/aws-perspective.tf.template @@ -7,30 +7,6 @@ resource "aws_lambda_layer_version" "python311_open_mpic_layer_{{region}}" { provider = aws.{{region}} } -resource "aws_lambda_layer_version" "mpic_common_layer_{{region}}" { - filename = "../layer/mpic_common_layer_content.zip" - layer_name = "mpic_common_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_common_layer_content.zip")}" - compatible_runtimes = ["python3.11"] - provider = aws.{{region}} -} - -resource "aws_lambda_layer_version" "mpic_caa_checker_layer_{{region}}" { - filename = "../layer/mpic_caa_checker_layer_content.zip" - layer_name = "mpic_caa_checker_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_caa_checker_layer_content.zip")}" - compatible_runtimes = ["python3.11"] - provider = aws.{{region}} -} - -resource "aws_lambda_layer_version" "mpic_dcv_checker_layer_{{region}}" { - filename = "../layer/mpic_dcv_checker_layer_content.zip" - layer_name = "mpic_dcv_checker_layer_{{region}}_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_dcv_checker_layer_content.zip")}" - compatible_runtimes = ["python3.11"] - provider = aws.{{region}} -} - variable "vpc_cidr_block_{{region}}" { type = string description = "VPC CIDR" @@ -50,6 +26,7 @@ variable "subnet_private_cidr_block_{{region}}" { } resource "aws_vpc" "vpc_{{region}}" { + count = var.dnssec_enabled ? 1 : 0 cidr_block = var.vpc_cidr_block_{{region}} tags = { Name = "mpic-{{region}}-vpc" @@ -60,7 +37,8 @@ resource "aws_vpc" "vpc_{{region}}" { } resource "aws_subnet" "subnet_public_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id cidr_block = var.subnet_public_cidr_block_{{region}} depends_on = [aws_vpc.vpc_{{region}}] map_public_ip_on_launch = true @@ -71,7 +49,8 @@ resource "aws_subnet" "subnet_public_{{region}}" { } resource "aws_internet_gateway" "internet_gateway_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id tags = { Name = "mpic-{{region}}-internet-gateway" @@ -80,11 +59,12 @@ resource "aws_internet_gateway" "internet_gateway_{{region}}" { } resource "aws_route_table" "route_table_public_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id route { cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.internet_gateway_{{region}}.id + gateway_id = aws_internet_gateway.internet_gateway_{{region}}[count.index].id } tags = { @@ -94,12 +74,14 @@ resource "aws_route_table" "route_table_public_{{region}}" { } resource "aws_route_table_association" "route_table_association_public_{{region}}" { - subnet_id = aws_subnet.subnet_public_{{region}}.id - route_table_id = aws_route_table.route_table_public_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + subnet_id = aws_subnet.subnet_public_{{region}}[count.index].id + route_table_id = aws_route_table.route_table_public_{{region}}[count.index].id provider = aws.{{region}} } resource "aws_eip" "eip_{{region}}" { + count = var.dnssec_enabled ? 1 : 0 domain = "vpc" depends_on = [aws_internet_gateway.internet_gateway_{{region}}] tags = { @@ -109,8 +91,9 @@ resource "aws_eip" "eip_{{region}}" { } resource "aws_nat_gateway" "nat_gateway_{{region}}" { - allocation_id = aws_eip.eip_{{region}}.id - subnet_id = aws_subnet.subnet_public_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + allocation_id = aws_eip.eip_{{region}}[count.index].id + subnet_id = aws_subnet.subnet_public_{{region}}[count.index].id tags = { Name = "mpic-{{region}}-nat-gateway" @@ -119,7 +102,8 @@ resource "aws_nat_gateway" "nat_gateway_{{region}}" { } resource "aws_subnet" "subnet_private_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id cidr_block = var.subnet_private_cidr_block_{{region}} map_public_ip_on_launch = false depends_on = [aws_vpc.vpc_{{region}}] @@ -129,13 +113,13 @@ resource "aws_subnet" "subnet_private_{{region}}" { provider = aws.{{region}} } - resource "aws_route_table" "route_table_private_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id route { cidr_block = "0.0.0.0/0" - nat_gateway_id = aws_nat_gateway.nat_gateway_{{region}}.id + nat_gateway_id = aws_nat_gateway.nat_gateway_{{region}}[count.index].id } tags = { @@ -145,14 +129,16 @@ resource "aws_route_table" "route_table_private_{{region}}" { } resource "aws_route_table_association" "route_table_association_private_{{region}}" { - subnet_id = aws_subnet.subnet_private_{{region}}.id - route_table_id = aws_route_table.route_table_private_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + subnet_id = aws_subnet.subnet_private_{{region}}[count.index].id + route_table_id = aws_route_table.route_table_private_{{region}}[count.index].id provider = aws.{{region}} } resource "aws_default_network_acl" "default_network_acl_{{region}}" { - default_network_acl_id = aws_vpc.vpc_{{region}}.default_network_acl_id - subnet_ids = [aws_subnet.subnet_public_{{region}}.id, aws_subnet.subnet_private_{{region}}.id] + count = var.dnssec_enabled ? 1 : 0 + default_network_acl_id = aws_vpc.vpc_{{region}}[count.index].default_network_acl_id + subnet_ids = [aws_subnet.subnet_public_{{region}}[count.index].id, aws_subnet.subnet_private_{{region}}[count.index].id] ingress { protocol = -1 @@ -178,9 +164,9 @@ resource "aws_default_network_acl" "default_network_acl_{{region}}" { provider = aws.{{region}} } - resource "aws_default_security_group" "default_security_group_{{region}}" { - vpc_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + vpc_id = aws_vpc.vpc_{{region}}[count.index].id ingress { protocol = -1 @@ -194,7 +180,7 @@ resource "aws_default_security_group" "default_security_group_{{region}}" { to_port = 0 protocol = "-1" cidr_blocks = ["0.0.0.0/0"] - # cidr_blocks = ["127.0.0.1/32"] + # cidr_blocks = ["127.0.0.1/32"] } tags = { @@ -204,7 +190,8 @@ resource "aws_default_security_group" "default_security_group_{{region}}" { } resource "aws_route53_resolver_dnssec_config" "dnssec_config_{{region}}" { - resource_id = aws_vpc.vpc_{{region}}.id + count = var.dnssec_enabled ? 1 : 0 + resource_id = aws_vpc.vpc_{{region}}[count.index].id provider = aws.{{region}} } @@ -212,7 +199,12 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { filename = "../{{source-path}}/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda.zip" function_name = "open_mpic_dcv_checker_lambda_{{region}}_{{deployment-id}}" role = aws_iam_role.open_mpic_lambda_role.arn - depends_on = [aws_iam_role.open_mpic_lambda_role, aws_iam_role_policy_attachment.basic-execution-policy-attach, aws_iam_role_policy_attachment.vpc-policy-attach, aws_iam_role_policy_attachment.invoke-lambda-policy-attach] + depends_on = [ + aws_iam_role.open_mpic_lambda_role, + aws_iam_role_policy_attachment.basic-execution-policy-attach, + aws_iam_role_policy_attachment.invoke-lambda-policy-attach, + aws_iam_role_policy_attachment.vpc-policy-attach + ] handler = "mpic_dcv_checker_lambda_function.lambda_handler" source_code_hash = filebase64sha256("../{{source-path}}/mpic_dcv_checker_lambda/mpic_dcv_checker_lambda.zip") timeout = 60 @@ -220,8 +212,6 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { architectures = ["arm64"] layers = [ aws_lambda_layer_version.python311_open_mpic_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_common_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_dcv_checker_layer_{{region}}.arn, ] environment { variables = { @@ -229,8 +219,8 @@ resource "aws_lambda_function" "mpic_dcv_checker_lambda_{{region}}" { } } vpc_config { - subnet_ids = [aws_subnet.subnet_private_{{region}}.id] - security_group_ids = [aws_default_security_group.default_security_group_{{region}}.id] + subnet_ids = [for s in aws_subnet.subnet_private_{{region}} : s.id] + security_group_ids = [for s in aws_default_security_group.default_security_group_{{region}} : s.id] } provider = aws.{{region}} } @@ -239,7 +229,12 @@ resource "aws_lambda_function" "mpic_caa_checker_lambda_{{region}}" { filename = "../{{source-path}}/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip" function_name = "open_mpic_caa_checker_lambda_{{region}}_{{deployment-id}}" role = aws_iam_role.open_mpic_lambda_role.arn - depends_on = [aws_iam_role.open_mpic_lambda_role, aws_iam_role_policy_attachment.basic-execution-policy-attach, aws_iam_role_policy_attachment.vpc-policy-attach, aws_iam_role_policy_attachment.invoke-lambda-policy-attach] + depends_on = [ + aws_iam_role.open_mpic_lambda_role, + aws_iam_role_policy_attachment.basic-execution-policy-attach, + aws_iam_role_policy_attachment.invoke-lambda-policy-attach, + aws_iam_role_policy_attachment.vpc-policy-attach + ] handler = "mpic_caa_checker_lambda_function.lambda_handler" source_code_hash = filebase64sha256("../{{source-path}}/mpic_caa_checker_lambda/mpic_caa_checker_lambda.zip") timeout = 60 @@ -247,12 +242,10 @@ resource "aws_lambda_function" "mpic_caa_checker_lambda_{{region}}" { architectures = ["arm64"] layers = [ aws_lambda_layer_version.python311_open_mpic_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_common_layer_{{region}}.arn, - aws_lambda_layer_version.mpic_caa_checker_layer_{{region}}.arn, ] vpc_config { - subnet_ids = [aws_subnet.subnet_private_{{region}}.id] - security_group_ids = [aws_default_security_group.default_security_group_{{region}}.id] + subnet_ids = [for s in aws_subnet.subnet_private_{{region}} : s.id] + security_group_ids = [for s in aws_default_security_group.default_security_group_{{region}} : s.id] } provider = aws.{{region}} environment { diff --git a/open-tofu/main.tf.template b/open-tofu/main.tf.template index 88697c1..e6cbd4e 100644 --- a/open-tofu/main.tf.template +++ b/open-tofu/main.tf.template @@ -12,14 +12,6 @@ resource "aws_lambda_layer_version" "python311_open_mpic_layer" { compatible_runtimes = ["python3.11"] } -# Mpic Common layer for all lambdas (contains supporting first-party source code) -resource "aws_lambda_layer_version" "mpic_common_layer" { - filename = "../layer/mpic_common_layer_content.zip" - layer_name = "mpic_common_layer_{{deployment-id}}" - source_code_hash = "${filebase64sha256("../layer/mpic_common_layer_content.zip")}" - compatible_runtimes = ["python3.11"] -} - # Mpic Coordinator layer for the mpic coordinator lambda (contains supporting first-party source code) resource "aws_lambda_layer_version" "mpic_coordinator_layer" { filename = "../layer/mpic_coordinator_layer_content.zip" @@ -61,6 +53,7 @@ resource "aws_iam_role_policy_attachment" "basic-execution-policy-attach" { } resource "aws_iam_role_policy_attachment" "vpc-policy-attach" { + count = var.dnssec_enabled ? 1 : 0 role = "${aws_iam_role.open_mpic_lambda_role.name}" policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" } @@ -83,7 +76,6 @@ resource "aws_lambda_function" "mpic_coordinator_lambda" { timeout = 60 layers = [ aws_lambda_layer_version.python311_open_mpic_layer.arn, - aws_lambda_layer_version.mpic_common_layer.arn, aws_lambda_layer_version.mpic_coordinator_layer.arn, ] environment { diff --git a/open-tofu/variables.tf b/open-tofu/variables.tf new file mode 100644 index 0000000..05a1c85 --- /dev/null +++ b/open-tofu/variables.tf @@ -0,0 +1,5 @@ +variable "dnssec_enabled" { + type = bool + description = "Enable DNSSEC" + default = true +} diff --git a/pyproject.toml b/pyproject.toml index 48571f9..612d81e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "dnspython==2.6.1", "pydantic==2.8.2", "aws-lambda-powertools[parser]==3.2.0", - "open-mpic-core==2.2.0" + "open-mpic-core==2.2.0", ] [project.optional-dependencies] @@ -53,8 +53,8 @@ Source = "https://github.com/open-mpic/aws-lambda-python" [tool.hatch] version.path = "src/aws_lambda_mpic/__about__.py" -build.sources = ["src", "resources", "open_mpic_core_python/src"] -build.targets.wheel.packages = ["src/aws_lambda_mpic", "open_mpic_core_python/src/open_mpic_core"] +build.sources = ["src", "resources"] +build.targets.wheel.packages = ["src/aws_lambda_mpic"] [tool.hatch.envs.default] skip-install = false @@ -66,29 +66,61 @@ PIP_INDEX_URL = "https://pypi.org/simple/" PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" # FIXME here temporarily to test open-mpic-core packaging PIP_VERBOSE = "1" +[tool.hatch.envs.lambda-layer] +skip-install = true +python = "3.11" +type="virtual" +path="layer/create_layer_virtualenv" + +[tool.hatch.envs.lambda-layer.env-vars] +PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" +PIP_ONLY_BINARY = ":all:" +#PIP_PLATFORM = "manylinux2014_aarch64" +#PIP_TARGET = "layer/create_layer_virtualenv2/lib/python3.11/site-packages" # does not work... bug in pip 24.2? + +[tool.hatch.envs.lambda-layer.scripts] +install = "pip install . --platform manylinux2014_aarch64 --only-binary=:all: --target layer/create_layer_virtualenv/lib/python3.11/site-packages" + [tool.hatch.envs.test] +skip-install = false features = [ "test", "provided" ] +installer = "pip" + +[tool.hatch.envs.test.env-vars] +PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" [tool.hatch.envs.test.scripts] +pre-install = "python -m ensurepip" unit = "pytest" unit-html = "pytest --html=testreports/index.html" # generate html report (warning: uses an aging plugin, 11-2023) integration = "pytest tests/integration" coverage = "pytest --cov=src/aws_lambda_mpic --cov-report=term-missing --cov-report=html" [tool.hatch.envs.hatch-test] -features = [ - "test", - "provided" -] +#installer = "pip" +#features = [ +# "test", +# "provided" +#] +#dependencies = [ +# "pytest==8.2.2", +# "pytest-cov==5.0.0", +# "pytest-mock==3.14.0", +# "pytest-html==4.1.1", +# "pytest-spec==3.2.0", +#] default-args = ["tests/unit"] randomize = true +#[tool.hatch.envs.hatch-test.env-vars] +#PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" + [tool.pytest.ini_options] pythonpath = [ - "src", "tests", "open_mpic_core_python/src", "." # need root directory because it has some python utility files we are importing for integration tests + "src", "tests", "." # need root directory because it has some python utility files we are importing for integration tests ] testpaths = [ "tests/unit" diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index 7e3895f..7e7a9fd 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -17,6 +17,7 @@ MPIC_REQUEST_PATH = "/mpic" + # noinspection PyMethodMayBeStatic @pytest.mark.integration class TestDeployedMpicApi: @@ -55,25 +56,25 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se @pytest.mark.parametrize('domain_or_ip_target, purpose_of_test, is_wildcard_domain', [ ('empty.basic.caatestsuite.com', 'Tests handling of 0 issue ";"', False), ('deny.basic.caatestsuite.com', 'Tests handling of 0 issue "caatestsuite.com"', False), - ('uppercase-deny.basic.caatestsuite.com', 'Tests handling of uppercase issue tag (0 ISSUE "caatestsuite.com")', False), - ('mixedcase-deny.basic.caatestsuite.com', 'Tests handling of mixed case issue tag (0 IsSuE "caatestsuite.com")', False), - ('big.basic.caatestsuite.com', 'Tests handling of gigantic (1001) CAA record set (0 issue "caatestsuite.com")', False), - ('critical1.basic.caatestsuite.com', 'Tests handling of unknown critical property (128 caatestsuitedummyproperty "test")', False), - ('critical2.basic.caatestsuite.com', 'Tests handling of unknown critical property with another flag (130)', False), - ('sub1.deny.basic.caatestsuite.com', 'Tests basic tree climbing when CAA record is at parent domain', False), - ('sub2.sub1.deny.basic.caatestsuite.com', 'Tests tree climbing when CAA record is at grandparent domain', False), - ('deny.basic.caatestsuite.com', 'Tests handling of issue property for a wildcard domain', True), - ('deny-wild.basic.caatestsuite.com', 'Tests handling of issuewild for a wildcard domain', True), - ('cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME, where CAA record is at CNAME target', False), - ('cname-cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME chain, where CAA record is at ultimate target', False), - ('sub1.cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME, where parent is CNAME and CAA record is at target', False), + # ('uppercase-deny.basic.caatestsuite.com', 'Tests handling of uppercase issue tag (0 ISSUE "caatestsuite.com")', False), + # ('mixedcase-deny.basic.caatestsuite.com', 'Tests handling of mixed case issue tag (0 IsSuE "caatestsuite.com")', False), + # ('big.basic.caatestsuite.com', 'Tests handling of gigantic (1001) CAA record set (0 issue "caatestsuite.com")', False), + # ('critical1.basic.caatestsuite.com', 'Tests handling of unknown critical property (128 caatestsuitedummyproperty "test")', False), + # ('critical2.basic.caatestsuite.com', 'Tests handling of unknown critical property with another flag (130)', False), + # ('sub1.deny.basic.caatestsuite.com', 'Tests basic tree climbing when CAA record is at parent domain', False), + # ('sub2.sub1.deny.basic.caatestsuite.com', 'Tests tree climbing when CAA record is at grandparent domain', False), + # ('deny.basic.caatestsuite.com', 'Tests handling of issue property for a wildcard domain', True), + # ('deny-wild.basic.caatestsuite.com', 'Tests handling of issuewild for a wildcard domain', True), + # ('cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME, where CAA record is at CNAME target', False), + # ('cname-cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME chain, where CAA record is at ultimate target', False), + # ('sub1.cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME, where parent is CNAME and CAA record is at target', False), ('deny.permit.basic.caatestsuite.com', 'Tests rejection when parent name contains a permissible CAA record set', False), - ('ipv6only.caatestsuite.com', 'Tests handling of record at IPv6-only authoritative name server', False), - ('expired.caatestsuite-dnssec.com', 'Tests rejection when expired DNSSEC signatures', False), - ('missing.caatestsuite-dnssec.com', 'Tests rejection when missing DNSSEC signatures', False), - ('blackhole.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to non-responsive server', False), - ('servfail.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning SERVFAIL', False), - ('refused.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning REFUSED', False), + # ('ipv6only.caatestsuite.com', 'Tests handling of record at IPv6-only authoritative name server', False), + # ('expired.caatestsuite-dnssec.com', 'Tests rejection when expired DNSSEC signatures', False), + # ('missing.caatestsuite-dnssec.com', 'Tests rejection when missing DNSSEC signatures', False), + # ('blackhole.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to non-responsive server', False), + # ('servfail.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning SERVFAIL', False), + # ('refused.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning REFUSED', False), ('xss.caatestsuite.com', 'Tests rejection when issue property has HTML and JS', False), ]) def api_should_return_is_valid_false_for_all_tests_in_do_not_issue_caa_test_suite(self, api_client, domain_or_ip_target, @@ -94,6 +95,7 @@ def api_should_return_is_valid_false_for_all_tests_in_do_not_issue_caa_test_suit # This case is handled in a compliant manner as it is treated as a lookup failure. # The test for proper communication with an IPv6 nameserver can be enabled with the following additional parameter to the list below. # ('ipv6only.caatestsuite.com', 'Tests handling of record at IPv6-only authoritative name server', False), + @pytest.mark.skip(reason='need faster feedback loop for now') @pytest.mark.parametrize('domain_or_ip_target, purpose_of_test, is_wildcard_domain', [ ('deny.basic.caatestsuite.com', 'Tests handling of 0 issue "caatestsuite.com"', False), ('uppercase-deny.basic.caatestsuite.com', 'Tests handling of uppercase issue tag (0 ISSUE "caatestsuite.com")', False), @@ -146,7 +148,7 @@ def api_should_return_200_given_valid_dcv_validation(self, api_client): orchestration_parameters=MpicRequestOrchestrationParameters(perspective_count=3, quorum_count=2), dcv_check_parameters=DcvCheckParameters( validation_details=DcvWebsiteChangeValidationDetails(http_token_path='/', - challenge_value='test') + challenge_value='test') ) ) @@ -162,7 +164,7 @@ def api_should_return_200_and_failed_corroboration_given_failed_dcv_check(self, domain_or_ip_target='ifconfig.me', dcv_check_parameters=DcvCheckParameters( validation_details=DcvWebsiteChangeValidationDetails(http_token_path='/', - challenge_value='test') + challenge_value='test') ) ) @@ -186,7 +188,7 @@ def api_should_return_500_given_invalid_orchestration_parameters_in_request(self print("\nResponse:\n", json.dumps(response_body, indent=4)) # pretty print response body assert response_body['error'] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key # We lost error detail in the last push. - #assert any(issue['issue_type'] == MpicRequestValidationMessages.INVALID_QUORUM_COUNT.key for issue in response_body['validation_issues']) + # assert any(issue['issue_type'] == MpicRequestValidationMessages.INVALID_QUORUM_COUNT.key for issue in response_body['validation_issues']) def api_should_return_502_given_invalid_check_type_in_request(self, api_client): request = MpicCaaRequest( @@ -199,7 +201,7 @@ def api_should_return_502_given_invalid_check_type_in_request(self, api_client): print("\nRequest:\n", json.dumps(request.model_dump(), indent=4)) # pretty print request body response = api_client.post(MPIC_REQUEST_PATH, json.dumps(request.model_dump())) assert response.status_code == 502 - #response_body = json.loads(response.text) - #print("\nResponse:\n", json.dumps(response_body, indent=4)) + # response_body = json.loads(response.text) + # print("\nResponse:\n", json.dumps(response_body, indent=4)) # We last error details in the last push. - #assert response_body['error'] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key + # assert response_body['error'] == MpicRequestValidationMessages.REQUEST_VALIDATION_FAILED.key From 6dd9fbf3dff614aa8757091a2d82360004794197 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 10 Dec 2024 16:55:06 -0700 Subject: [PATCH 3/9] removed requirements.txt and renamed 2-package to just package --- layer/1-install.sh | 5 ----- layer/{2-package.sh => package.sh} | 0 layer/requirements.txt.backup | 5 ----- 3 files changed, 10 deletions(-) delete mode 100755 layer/1-install.sh rename layer/{2-package.sh => package.sh} (100%) delete mode 100644 layer/requirements.txt.backup diff --git a/layer/1-install.sh b/layer/1-install.sh deleted file mode 100755 index 3d0a538..0000000 --- a/layer/1-install.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -python3.11 -m venv --clear create_layer_virtualenv -source create_layer_virtualenv/bin/activate -# need to explicitly set target directory to install dependencies when explicitly specifying platform -pip install -r requirements.txt --platform manylinux2014_aarch64 --only-binary=:all: --target "$VIRTUAL_ENV/lib/python3.11/site-packages" diff --git a/layer/2-package.sh b/layer/package.sh similarity index 100% rename from layer/2-package.sh rename to layer/package.sh diff --git a/layer/requirements.txt.backup b/layer/requirements.txt.backup deleted file mode 100644 index d0c479a..0000000 --- a/layer/requirements.txt.backup +++ /dev/null @@ -1,5 +0,0 @@ -requests>=2.32.0 -dnspython==2.6.1 -pydantic==2.8.2 -pyyaml==6.0.1 -aws-lambda-powertools==3.2.0 \ No newline at end of file From 97f64c1024f238eea11d84702c667ae11739d8e2 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Tue, 10 Dec 2024 16:57:49 -0700 Subject: [PATCH 4/9] updated README to reflect new deployment steps --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 088159c..db192f4 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,14 @@ All requirements for running the API are packaged and uploaded to AWS as a lambd - Hatch (https://hatch.pypa.io/) for building and running the project. This is a Python project manager that can be installed via `pip install hatch`. ## Deployment Steps -1. Init and update the git submodule using: `git submodule init` followed by `git submodule update` in the root of the project. -2. Install layer dependencies. cd to the `layer` directory. Run `./1-install.sh` to create a virtual Python environment and install the project dependencies via pip. -3. Package the AWS layer. In the `layer` directory, run `./2-package.sh`. This will make a file called `layer_content.zip` which will later be referenced by Open Tofu. -4. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` -5. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. -6. Run `hatch run ./configure.py` from the root directory of the repo to generate Open Tofu files from templates. -7. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. -8. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. -9. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. +1. From the project root directory, run `hatch env lambda-layer:install`. This will create a virtual Python environment in the `layer` directory and install the project dependencies via pip. +2. Package the AWS layer. In the `layer` directory, run `./package.sh`. This will make two files: `python311_layer_content.zip` and `mpic_coordinator_layer_content.zip` which will later be referenced by Open Tofu. +3. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` +4. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. +5. Run `hatch run ./configure.py` from the root directory of the repo to generate Open Tofu files from templates. +6. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. +7. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. +8. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. ## Testing The following is an example of a test API call that uses bash command substitution to fill in the proper values for the API URL and the API key. From 3fd22c990d3a83e41b553599fc6ef24da06442de Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Wed, 11 Dec 2024 16:37:27 -0700 Subject: [PATCH 5/9] removed some commented-out lines --- pyproject.toml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 612d81e..144a261 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,24 +100,9 @@ integration = "pytest tests/integration" coverage = "pytest --cov=src/aws_lambda_mpic --cov-report=term-missing --cov-report=html" [tool.hatch.envs.hatch-test] -#installer = "pip" -#features = [ -# "test", -# "provided" -#] -#dependencies = [ -# "pytest==8.2.2", -# "pytest-cov==5.0.0", -# "pytest-mock==3.14.0", -# "pytest-html==4.1.1", -# "pytest-spec==3.2.0", -#] default-args = ["tests/unit"] randomize = true -#[tool.hatch.envs.hatch-test.env-vars] -#PIP_EXTRA_INDEX_URL = "https://test.pypi.org/simple/" - [tool.pytest.ini_options] pythonpath = [ "src", "tests", "." # need root directory because it has some python utility files we are importing for integration tests From f039f350c150cf501f46a6258cd38475e3d479c0 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Wed, 11 Dec 2024 22:33:29 -0700 Subject: [PATCH 6/9] removed pytest-spec because it's GPL licensed --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 144a261..1bd4198 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ test = [ "pytest-cov==5.0.0", "pytest-mock==3.14.0", "pytest-html==4.1.1", - "pytest-spec==3.2.0", ] [project.urls] @@ -119,7 +118,6 @@ markers = [ ] addopts = [ "--import-mode=prepend", # explicit default, as the tests rely on it for proper import resolution - "--spec" # show test names in a more readable format in the console (warning: uses an aging plugin, 5-2021) ] spec_header_format = "Spec for {test_case} ({path}):" spec_test_format = "{result} {docstring_summary}" # defaults to {name} if docstring is not present in test From 97d6ccbf5acbb7008c478f41ae07b6b4b42652a4 Mon Sep 17 00:00:00 2001 From: Dmitry Sharkov Date: Thu, 12 Dec 2024 19:07:38 -0500 Subject: [PATCH 7/9] fixed the hatch instruction from env to run --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db192f4..1df7e28 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ All requirements for running the API are packaged and uploaded to AWS as a lambd - Hatch (https://hatch.pypa.io/) for building and running the project. This is a Python project manager that can be installed via `pip install hatch`. ## Deployment Steps -1. From the project root directory, run `hatch env lambda-layer:install`. This will create a virtual Python environment in the `layer` directory and install the project dependencies via pip. +1. From the project root directory, run `hatch run lambda-layer:install`. This will create a virtual Python environment in the `layer` directory and install the project dependencies via pip. 2. Package the AWS layer. In the `layer` directory, run `./package.sh`. This will make two files: `python311_layer_content.zip` and `mpic_coordinator_layer_content.zip` which will later be referenced by Open Tofu. 3. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` 4. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. From a96150690d6e5b54a3b6cec9ae1d0b5edbc09d6b Mon Sep 17 00:00:00 2001 From: Henry Birge-Lee Date: Thu, 12 Dec 2024 21:48:30 -0500 Subject: [PATCH 8/9] improved --- README.md | 6 ++++-- deploy.sh | 2 +- open_mpic_core_python | 1 + tests/integration/test_deployed_mpic_api.py | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) create mode 160000 open_mpic_core_python diff --git a/README.md b/README.md index 1df7e28..15a7c30 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,12 @@ All requirements for running the API are packaged and uploaded to AWS as a lambd 3. Zip all functions. AWS Lambda functions are usually deployed from zip files. cd to the main project directory and then run `./zip-all.sh` 4. Create `config.yaml` in the root directory of the repo to contain the proper values needed for the deployment. A default config.yaml for a 6-perspective deployment with the controller in us-east-2 is included in this repo as `config.example.yaml`. This config can be made the active config by running `cp config.example.yaml config.yaml` in the root directory. 5. Run `hatch run ./configure.py` from the root directory of the repo to generate Open Tofu files from templates. -6. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. +6. Deploy the entire package with Open Tofu. cd to the `open-tofu` directory where .tf files are located. Then run `tofu init`. Then run `tofu apply` and type `yes` at the confirmation prompt. This provides a standard install with DNSSEC enabled which causes the system to incur expenses even when it is not in use (due to the AWS VPC NAT Gateways needed). To reduce the AWS bill, DNSSEC can also be disabled by appending `-var="dnssec_enabled=false"` to `tofu apply` (i.e., `tofu apply -var="dnssec_enabled=false"`). 7. Get the URL of the deployed API endpoint by running `hatch run ./get_api_url.py` in the root directory. 8. Get the API Key generated by AWS by running `hatch run ./get_api_key.py` in the root directory. The deployment is configured to reject any API call that does not have this key passed via the `x-api-key` HTTP header. +For convenience `./deploy.sh` in the project root will perform all of these steps (using `-var="dnssec_enabled=false"`) with the exception of copying over the example config to the operational config and running `tofu init` in the open-tofu dir. + ## Testing The following is an example of a test API call that uses bash command substitution to fill in the proper values for the API URL and the API key. @@ -39,7 +41,7 @@ The above sample must be run from the root directory of a deployed Open MPIC aws The API is compliant with the [Open MPIC Specification](https://github.com/open-mpic/open-mpic-specification). -Documentation based on the API specification can be viewed [here](https://open-mpic.org/documentation.html). +Documentation based on the API specification used in this version can be viewed [here](https://open-mpic.org/documentation.html?commit=f763382c38a867dda3253afded017f9e3a24ead5). ## Development Code changes can easily be deployed by editing the .py files and then rezipping the project via `./zip-all.sh` and `./2-package.sh` in the `layer` directory. Then, running `tofu apply` run from the open-tofu directory will update only on the required resources and leave the others unchanged. If any `.tf.template` files are changed or `config.yaml` is edited, `hatch run ./configure.py` must be rerun followed by `tofu apply` in the open-tofu directory. diff --git a/deploy.sh b/deploy.sh index 9437a95..3d0de2b 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,2 +1,2 @@ #!/bin/bash -./clean.sh; cd layer; ./1-install.sh; ./2-package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -auto-approve; cd .. +./clean.sh; hatch run lambda-layer:install; cd layer; ./package.sh; cd ..; hatch run ./configure.py; ./zip-all.sh; cd open-tofu; tofu apply -var="dnssec_enabled=false" -auto-approve; cd .. diff --git a/open_mpic_core_python b/open_mpic_core_python new file mode 160000 index 0000000..e758443 --- /dev/null +++ b/open_mpic_core_python @@ -0,0 +1 @@ +Subproject commit e7584433014eb701a2609e8e37af073c5575e9f7 diff --git a/tests/integration/test_deployed_mpic_api.py b/tests/integration/test_deployed_mpic_api.py index d34ba21..e4e6999 100644 --- a/tests/integration/test_deployed_mpic_api.py +++ b/tests/integration/test_deployed_mpic_api.py @@ -70,8 +70,8 @@ def api_should_return_200_and_passed_corroboration_given_successful_caa_check(se ('sub1.cname-deny.basic.caatestsuite.com', 'Tests handling of CNAME, where parent is CNAME and CAA record is at target', False), ('deny.permit.basic.caatestsuite.com', 'Tests rejection when parent name contains a permissible CAA record set', False), ('ipv6only.caatestsuite.com', 'Tests handling of record at IPv6-only authoritative name server', False), - ('expired.caatestsuite-dnssec.com', 'Tests rejection when expired DNSSEC signatures', False), - ('missing.caatestsuite-dnssec.com', 'Tests rejection when missing DNSSEC signatures', False), + #('expired.caatestsuite-dnssec.com', 'Tests rejection when expired DNSSEC signatures', False), # DNSSEC SHOULD be enabled in production but is not a current requirement for MPIC + #('missing.caatestsuite-dnssec.com', 'Tests rejection when missing DNSSEC signatures', False), # DNSSEC SHOULD be enabled in production but is not a current requirement for MPIC ('blackhole.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to non-responsive server', False), ('servfail.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning SERVFAIL', False), ('refused.caatestsuite-dnssec.com', 'Tests rejection when DNSSEC chain goes to server returning REFUSED', False), From 7181576f37423a42a2fdd87d12eff7524d8acdbf Mon Sep 17 00:00:00 2001 From: Henry Birge-Lee Date: Thu, 12 Dec 2024 21:56:14 -0500 Subject: [PATCH 9/9] cleanup --- open_mpic_core_python | 1 - 1 file changed, 1 deletion(-) delete mode 160000 open_mpic_core_python diff --git a/open_mpic_core_python b/open_mpic_core_python deleted file mode 160000 index e758443..0000000 --- a/open_mpic_core_python +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e7584433014eb701a2609e8e37af073c5575e9f7