From b23ac9ec09286a608ebd62fa93cfad487b5f0f6e Mon Sep 17 00:00:00 2001 From: Erik Zilber Date: Wed, 5 Feb 2025 14:53:31 -0500 Subject: [PATCH 01/10] Add support for Object Storage Gen 2 (#503) * wip * Added support for OBJ Gen 2 * Fix lint * Address PR comments * More PR comments --- linode_api4/groups/object_storage.py | 40 ++++++++++ linode_api4/objects/object_storage.py | 60 +++++++++++++-- .../object-storage_buckets_us-east-1.json | 4 +- ...rage_buckets_us-east-1_example-bucket.json | 4 +- ...buckets_us-east_example-bucket_access.json | 6 ++ .../models/object_storage/test_obj.py | 75 +++++++++++++++++-- test/unit/objects/object_storage_test.py | 27 +++++++ 7 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 test/fixtures/object-storage_buckets_us-east_example-bucket_access.json diff --git a/linode_api4/groups/object_storage.py b/linode_api4/groups/object_storage.py index f531932e0..f13237c58 100644 --- a/linode_api4/groups/object_storage.py +++ b/linode_api4/groups/object_storage.py @@ -5,6 +5,11 @@ from deprecated import deprecated +from linode_api4 import ( + ObjectStorageEndpoint, + ObjectStorageEndpointType, + PaginatedList, +) from linode_api4.errors import UnexpectedResponseError from linode_api4.groups import Group from linode_api4.objects import ( @@ -272,6 +277,30 @@ def transfer(self): return MappedObject(**result) + def endpoints(self, *filters) -> PaginatedList: + """ + Returns a paginated list of all Object Storage endpoints available in your account. + + This is intended to be called from the :any:`LinodeClient` + class, like this:: + + endpoints = client.object_storage.endpoints() + + API Documentation: https://techdocs.akamai.com/linode-api/reference/get-object-storage-endpoints + + :param filters: Any number of filters to apply to this query. + See :doc:`Filtering Collections` + for more details on filtering. + + :returns: A list of Object Storage Endpoints that matched the query. + :rtype: PaginatedList of ObjectStorageEndpoint + """ + return self.client._get_and_filter( + ObjectStorageEndpoint, + *filters, + endpoint="/object-storage/endpoints", + ) + def buckets(self, *filters): """ Returns a paginated list of all Object Storage Buckets that you own. @@ -299,6 +328,8 @@ def bucket_create( label: str, acl: ObjectStorageACL = ObjectStorageACL.PRIVATE, cors_enabled=False, + s3_endpoint: Optional[str] = None, + endpoint_type: Optional[ObjectStorageEndpointType] = None, ): """ Creates an Object Storage Bucket in the specified cluster. Accounts with @@ -320,6 +351,13 @@ def bucket_create( should be created. :type cluster: str + :param endpoint_type: The type of s3_endpoint available to the active user in this region. + :type endpoint_type: str + Enum: E0,E1,E2,E3 + + :param s3_endpoint: The active user's s3 endpoint URL, based on the endpoint_type and region. + :type s3_endpoint: str + :param cors_enabled: If true, the bucket will be created with CORS enabled for all origins. For more fine-grained controls of CORS, use the S3 API directly. @@ -346,6 +384,8 @@ def bucket_create( "label": label, "acl": acl, "cors_enabled": cors_enabled, + "s3_endpoint": s3_endpoint, + "endpoint_type": endpoint_type, } if self.is_cluster(cluster_or_region_id): diff --git a/linode_api4/objects/object_storage.py b/linode_api4/objects/object_storage.py index f4ddfe9b5..76a3945e2 100644 --- a/linode_api4/objects/object_storage.py +++ b/linode_api4/objects/object_storage.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from typing import Optional from urllib import parse @@ -11,7 +12,7 @@ Property, Region, ) -from linode_api4.objects.serializable import StrEnum +from linode_api4.objects.serializable import JSONObject, StrEnum from linode_api4.util import drop_null_keys @@ -28,6 +29,27 @@ class ObjectStorageKeyPermission(StrEnum): READ_WRITE = "read_write" +class ObjectStorageEndpointType(StrEnum): + E0 = "E0" + E1 = "E1" + E2 = "E2" + E3 = "E3" + + +@dataclass +class ObjectStorageEndpoint(JSONObject): + """ + ObjectStorageEndpoint contains the core fields of an object storage endpoint object. + + NOTE: This is not implemented as a typical API object (Base) because Object Storage Endpoints + cannot be refreshed, as there is no singular GET endpoint. + """ + + region: str = "" + endpoint_type: ObjectStorageEndpointType = "" + s3_endpoint: Optional[str] = None + + class ObjectStorageBucket(DerivedBase): """ A bucket where objects are stored in. @@ -47,6 +69,8 @@ class ObjectStorageBucket(DerivedBase): "label": Property(identifier=True), "objects": Property(), "size": Property(), + "endpoint_type": Property(), + "s3_endpoint": Property(), } @classmethod @@ -63,13 +87,8 @@ def make_instance(cls, id, client, parent_id=None, json=None): Override this method to pass in the parent_id from the _raw_json object when it's available. """ - if json is None: - return None - - cluster_or_region = json.get("region") or json.get("cluster") - - if parent_id is None and cluster_or_region: - parent_id = cluster_or_region + if json is not None: + parent_id = parent_id or json.get("region") or json.get("cluster") if parent_id: return super().make(id, client, cls, parent_id=parent_id, json=json) @@ -78,6 +97,31 @@ def make_instance(cls, id, client, parent_id=None, json=None): "Unexpected json response when making a new Object Storage Bucket instance." ) + def access_get(self): + """ + Returns a result object which wraps the current access config for this ObjectStorageBucket. + + API Documentation: TODO + + :returns: A result object which wraps the access that this ObjectStorageBucket is currently configured with. + :rtype: MappedObject + """ + result = self._client.get( + "{}/access".format(self.api_endpoint), + model=self, + ) + + if not any( + key in result + for key in ["acl", "acl_xml", "cors_enabled", "cors_xml"] + ): + raise UnexpectedResponseError( + "Unexpected response when getting the bucket access config of a bucket!", + json=result, + ) + + return MappedObject(**result) + def access_modify( self, acl: Optional[ObjectStorageACL] = None, diff --git a/test/fixtures/object-storage_buckets_us-east-1.json b/test/fixtures/object-storage_buckets_us-east-1.json index f99a944a6..f1479dabb 100644 --- a/test/fixtures/object-storage_buckets_us-east-1.json +++ b/test/fixtures/object-storage_buckets_us-east-1.json @@ -6,7 +6,9 @@ "hostname": "example-bucket.us-east-1.linodeobjects.com", "label": "example-bucket", "objects": 4, - "size": 188318981 + "size": 188318981, + "endpoint_type": "E1", + "s3_endpoint": "us-east-12.linodeobjects.com" } ], "page": 1, diff --git a/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json b/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json index bb93ec99a..c9c6344ee 100644 --- a/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json +++ b/test/fixtures/object-storage_buckets_us-east-1_example-bucket.json @@ -5,5 +5,7 @@ "hostname": "example-bucket.us-east-1.linodeobjects.com", "label": "example-bucket", "objects": 4, - "size": 188318981 + "size": 188318981, + "endpoint_type": "E1", + "s3_endpoint": "us-east-12.linodeobjects.com" } \ No newline at end of file diff --git a/test/fixtures/object-storage_buckets_us-east_example-bucket_access.json b/test/fixtures/object-storage_buckets_us-east_example-bucket_access.json new file mode 100644 index 000000000..852803146 --- /dev/null +++ b/test/fixtures/object-storage_buckets_us-east_example-bucket_access.json @@ -0,0 +1,6 @@ +{ + "acl": "authenticated-read", + "acl_xml": "..." +} \ No newline at end of file diff --git a/test/integration/models/object_storage/test_obj.py b/test/integration/models/object_storage/test_obj.py index 82b2da022..0f3e39f33 100644 --- a/test/integration/models/object_storage/test_obj.py +++ b/test/integration/models/object_storage/test_obj.py @@ -8,6 +8,7 @@ ObjectStorageACL, ObjectStorageBucket, ObjectStorageCluster, + ObjectStorageEndpointType, ObjectStorageKeyPermission, ObjectStorageKeys, ) @@ -19,7 +20,14 @@ def region(test_linode_client: LinodeClient): @pytest.fixture(scope="session") -def bucket(test_linode_client: LinodeClient, region: str): +def endpoints(test_linode_client: LinodeClient): + return test_linode_client.object_storage.endpoints() + + +@pytest.fixture(scope="session") +def bucket( + test_linode_client: LinodeClient, region: str +) -> ObjectStorageBucket: bucket = test_linode_client.object_storage.bucket_create( cluster_or_region=region, label="bucket-" + str(time.time_ns()), @@ -31,6 +39,31 @@ def bucket(test_linode_client: LinodeClient, region: str): bucket.delete() +@pytest.fixture(scope="session") +def bucket_with_endpoint( + test_linode_client: LinodeClient, endpoints +) -> ObjectStorageBucket: + selected_endpoint = next( + ( + e + for e in endpoints + if e.endpoint_type == ObjectStorageEndpointType.E1 + ), + None, + ) + + bucket = test_linode_client.object_storage.bucket_create( + cluster_or_region=selected_endpoint.region, + label="bucket-" + str(time.time_ns()), + acl=ObjectStorageACL.PRIVATE, + cors_enabled=False, + endpoint_type=selected_endpoint.endpoint_type, + ) + + yield bucket + bucket.delete() + + @pytest.fixture(scope="session") def obj_key(test_linode_client: LinodeClient): key = test_linode_client.object_storage.keys_create( @@ -71,19 +104,39 @@ def test_keys( assert loaded_key.label == obj_key.label assert loaded_limited_key.label == obj_limited_key.label + assert ( + loaded_limited_key.regions[0].endpoint_type + in ObjectStorageEndpointType.__members__.values() + ) -def test_bucket( - test_linode_client: LinodeClient, - bucket: ObjectStorageBucket, -): - loaded_bucket = test_linode_client.load(ObjectStorageBucket, bucket.label) +def test_bucket(test_linode_client: LinodeClient, bucket: ObjectStorageBucket): + loaded_bucket = test_linode_client.load( + ObjectStorageBucket, + target_id=bucket.label, + target_parent_id=bucket.region, + ) assert loaded_bucket.label == bucket.label assert loaded_bucket.region == bucket.region -def test_bucket( +def test_bucket_with_endpoint( + test_linode_client: LinodeClient, bucket_with_endpoint: ObjectStorageBucket +): + loaded_bucket = test_linode_client.load( + ObjectStorageBucket, + target_id=bucket_with_endpoint.label, + target_parent_id=bucket_with_endpoint.region, + ) + + assert loaded_bucket.label == bucket_with_endpoint.label + assert loaded_bucket.region == bucket_with_endpoint.region + assert loaded_bucket.s3_endpoint is not None + assert loaded_bucket.endpoint_type == "E1" + + +def test_buckets_in_region( test_linode_client: LinodeClient, bucket: ObjectStorageBucket, region: str, @@ -103,6 +156,14 @@ def test_list_obj_storage_bucket( assert any(target_bucket_id == b.id for b in buckets) +def test_bucket_access_get(bucket: ObjectStorageBucket): + access = bucket.access_get() + + assert access.acl is not None + assert access.acl_xml is not None + assert access.cors_enabled is not None + + def test_bucket_access_modify(bucket: ObjectStorageBucket): bucket.access_modify(ObjectStorageACL.PRIVATE, cors_enabled=True) diff --git a/test/unit/objects/object_storage_test.py b/test/unit/objects/object_storage_test.py index 95d781a84..396813b3d 100644 --- a/test/unit/objects/object_storage_test.py +++ b/test/unit/objects/object_storage_test.py @@ -1,6 +1,7 @@ from datetime import datetime from test.unit.base import ClientBaseCase +from linode_api4 import ObjectStorageEndpointType from linode_api4.objects import ( ObjectStorageACL, ObjectStorageBucket, @@ -35,6 +36,14 @@ def test_object_storage_bucket_api_get(self): ) self.assertEqual(object_storage_bucket.objects, 4) self.assertEqual(object_storage_bucket.size, 188318981) + self.assertEqual( + object_storage_bucket.endpoint_type, + ObjectStorageEndpointType.E1, + ) + self.assertEqual( + object_storage_bucket.s3_endpoint, + "us-east-12.linodeobjects.com", + ) self.assertEqual(m.call_url, object_storage_bucket_api_get_url) def test_object_storage_bucket_delete(self): @@ -48,6 +57,22 @@ def test_object_storage_bucket_delete(self): object_storage_bucket.delete() self.assertEqual(m.call_url, object_storage_bucket_delete_url) + def test_bucket_access_get(self): + bucket_access_get_url = ( + "/object-storage/buckets/us-east/example-bucket/access" + ) + with self.mock_get(bucket_access_get_url) as m: + object_storage_bucket = ObjectStorageBucket( + self.client, "example-bucket", "us-east" + ) + result = object_storage_bucket.access_get() + self.assertIsNotNone(result) + self.assertEqual(m.call_url, bucket_access_get_url) + self.assertEqual(result.acl, "authenticated-read") + self.assertEqual(result.cors_enabled, True) + self.assertEqual(result.acl_xml, "...") + def test_bucket_access_modify(self): """ Test that you can modify bucket access settings. @@ -115,6 +140,8 @@ def test_buckets_in_cluster(self): self.assertEqual(bucket.label, "example-bucket") self.assertEqual(bucket.objects, 4) self.assertEqual(bucket.size, 188318981) + self.assertEqual(bucket.endpoint_type, ObjectStorageEndpointType.E1) + self.assertEqual(bucket.s3_endpoint, "us-east-12.linodeobjects.com") def test_ssl_cert_delete(self): """ From 5852396ed028f168628526c7dbb1ad05ea2e57e3 Mon Sep 17 00:00:00 2001 From: Lena Garber <114949949+lgarber-akamai@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:41:18 -0500 Subject: [PATCH 02/10] Deprecate methods and classes related to database backups (#506) * Deprecate DBaaS backup-related methods & classes * Fix import --- linode_api4/objects/database.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/linode_api4/objects/database.py b/linode_api4/objects/database.py index ea833eb8a..58044edb0 100644 --- a/linode_api4/objects/database.py +++ b/linode_api4/objects/database.py @@ -1,3 +1,5 @@ +from deprecated import deprecated + from linode_api4.objects import Base, DerivedBase, MappedObject, Property @@ -63,6 +65,9 @@ def invalidate(self): Base.invalidate(self) +@deprecated( + reason="Backups are not supported for non-legacy database clusters." +) class DatabaseBackup(DerivedBase): """ A generic Managed Database backup. @@ -97,6 +102,9 @@ def restore(self): ) +@deprecated( + reason="Backups are not supported for non-legacy database clusters." +) class MySQLDatabaseBackup(DatabaseBackup): """ A backup for an accessible Managed MySQL Database. @@ -107,6 +115,9 @@ class MySQLDatabaseBackup(DatabaseBackup): api_endpoint = "/databases/mysql/instances/{database_id}/backups/{id}" +@deprecated( + reason="Backups are not supported for non-legacy database clusters." +) class PostgreSQLDatabaseBackup(DatabaseBackup): """ A backup for an accessible Managed PostgreSQL Database. @@ -221,6 +232,9 @@ def patch(self): "{}/patch".format(MySQLDatabase.api_endpoint), model=self ) + @deprecated( + reason="Backups are not supported for non-legacy database clusters." + ) def backup_create(self, label, **kwargs): """ Creates a snapshot backup of a Managed MySQL Database. @@ -358,6 +372,9 @@ def patch(self): "{}/patch".format(PostgreSQLDatabase.api_endpoint), model=self ) + @deprecated( + reason="Backups are not supported for non-legacy database clusters." + ) def backup_create(self, label, **kwargs): """ Creates a snapshot backup of a Managed PostgreSQL Database. From b51634d8348ab2616cbfd1b03dd5534767a6a0d1 Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-akamai@users.noreply.github.com> Date: Mon, 10 Feb 2025 13:48:00 -0800 Subject: [PATCH 03/10] Add default string value for RUN_DB_TESTS (#509) --- .../models/database/test_database.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/test/integration/models/database/test_database.py b/test/integration/models/database/test_database.py index 5d8f74b41..d4f3c8796 100644 --- a/test/integration/models/database/test_database.py +++ b/test/integration/models/database/test_database.py @@ -130,7 +130,7 @@ def get_db_fork_status(): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_types(test_linode_client): @@ -142,7 +142,7 @@ def test_get_types(test_linode_client): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_engines(test_linode_client): @@ -156,7 +156,7 @@ def test_get_engines(test_linode_client): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_database_instance(test_linode_client, test_create_sql_db): @@ -167,7 +167,7 @@ def test_database_instance(test_linode_client, test_create_sql_db): # ------- POSTGRESQL DB Test cases ------- @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_sql_db_instance(test_linode_client, test_create_sql_db): @@ -185,7 +185,7 @@ def test_get_sql_db_instance(test_linode_client, test_create_sql_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_update_sql_db(test_linode_client, test_create_sql_db): @@ -218,7 +218,7 @@ def test_update_sql_db(test_linode_client, test_create_sql_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_sql_ssl(test_linode_client, test_create_sql_db): @@ -228,7 +228,7 @@ def test_get_sql_ssl(test_linode_client, test_create_sql_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_sql_patch(test_linode_client, test_create_sql_db): @@ -260,7 +260,7 @@ def test_sql_patch(test_linode_client, test_create_sql_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_sql_credentials(test_linode_client, test_create_sql_db): @@ -271,7 +271,7 @@ def test_get_sql_credentials(test_linode_client, test_create_sql_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_reset_sql_credentials(test_linode_client, test_create_sql_db): @@ -290,7 +290,7 @@ def test_reset_sql_credentials(test_linode_client, test_create_sql_db): # ------- POSTGRESQL DB Test cases ------- @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_postgres_db_instance(test_linode_client, test_create_postgres_db): @@ -310,7 +310,7 @@ def test_get_postgres_db_instance(test_linode_client, test_create_postgres_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_update_postgres_db(test_linode_client, test_create_postgres_db): @@ -345,7 +345,7 @@ def test_update_postgres_db(test_linode_client, test_create_postgres_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_postgres_ssl(test_linode_client, test_create_postgres_db): @@ -355,7 +355,7 @@ def test_get_postgres_ssl(test_linode_client, test_create_postgres_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_postgres_patch(test_linode_client, test_create_postgres_db): @@ -387,7 +387,7 @@ def test_postgres_patch(test_linode_client, test_create_postgres_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_get_postgres_credentials(test_linode_client, test_create_postgres_db): @@ -398,7 +398,7 @@ def test_get_postgres_credentials(test_linode_client, test_create_postgres_db): @pytest.mark.skipif( - os.getenv("RUN_DB_TESTS").strip().lower() not in {"yes", "true"}, + os.getenv("RUN_DB_TESTS", "").strip().lower() not in {"yes", "true"}, reason="RUN_DB_TESTS environment variable must be set to 'yes' or 'true' (case insensitive)", ) def test_reset_postgres_credentials( From f265d41368d3f4b90ce25123123df7b345d98b27 Mon Sep 17 00:00:00 2001 From: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> Date: Thu, 13 Feb 2025 01:48:17 -0500 Subject: [PATCH 04/10] Remove sensitive info output in testing (#512) --- test/integration/models/database/test_database.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/integration/models/database/test_database.py b/test/integration/models/database/test_database.py index d4f3c8796..1ad5dde3b 100644 --- a/test/integration/models/database/test_database.py +++ b/test/integration/models/database/test_database.py @@ -278,12 +278,9 @@ def test_reset_sql_credentials(test_linode_client, test_create_sql_db): db = test_linode_client.load(MySQLDatabase, test_create_sql_db.id) old_pass = str(db.credentials.password) - - print(old_pass) db.credentials_reset() time.sleep(5) - assert db.credentials.username == "akmadmin" assert db.credentials.password != old_pass From 4e590c11b9d51e582b8a3e5eb7f319b49cc116d6 Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-akamai@users.noreply.github.com> Date: Thu, 13 Feb 2025 14:12:46 -0800 Subject: [PATCH 05/10] add disabling acl tc (#511) --- test/integration/models/lke/test_lke.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/integration/models/lke/test_lke.py b/test/integration/models/lke/test_lke.py index f2fb3f2e5..7355ca40b 100644 --- a/test/integration/models/lke/test_lke.py +++ b/test/integration/models/lke/test_lke.py @@ -297,6 +297,22 @@ def test_lke_cluster_acl(lke_cluster_with_acl): assert acl == cluster.control_plane_acl assert acl.addresses.ipv4 == ["10.0.0.2/32"] + +def test_lke_cluster_disable_acl(lke_cluster_with_acl): + cluster = lke_cluster_with_acl + + assert cluster.control_plane_acl.enabled + + acl = cluster.control_plane_acl_update( + LKEClusterControlPlaneACLOptions( + enabled=False, + ) + ) + + assert acl.enabled is False + assert acl == cluster.control_plane_acl + assert acl.addresses.ipv4 == ["10.0.0.2/32"] + cluster.control_plane_acl_delete() assert not cluster.control_plane_acl.enabled From e5383773f7c31779e94acd31046a5d3fe08d84aa Mon Sep 17 00:00:00 2001 From: Lena Garber <114949949+lgarber-akamai@users.noreply.github.com> Date: Thu, 27 Feb 2025 10:55:38 -0500 Subject: [PATCH 06/10] Remove LKE ACL LA notices (#513) --- linode_api4/objects/lke.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/linode_api4/objects/lke.py b/linode_api4/objects/lke.py index e675eae8e..12d21f21f 100644 --- a/linode_api4/objects/lke.py +++ b/linode_api4/objects/lke.py @@ -79,8 +79,6 @@ class LKEClusterControlPlaneACLOptions(JSONObject): """ LKEClusterControlPlaneACLOptions is used to set the ACL configuration of an LKE cluster's control plane. - - NOTE: Control Plane ACLs may not currently be available to all users. """ enabled: Optional[bool] = None @@ -116,8 +114,6 @@ class LKEClusterControlPlaneACL(JSONObject): """ LKEClusterControlPlaneACL describes the ACL configuration of an LKE cluster's control plane. - - NOTE: Control Plane ACLs may not currently be available to all users. """ include_none_values = True @@ -337,8 +333,6 @@ def control_plane_acl(self) -> LKEClusterControlPlaneACL: """ Gets the ACL configuration of this cluster's control plane. - NOTE: Control Plane ACLs may not currently be available to all users. - API Documentation: https://techdocs.akamai.com/linode-api/reference/get-lke-cluster-acl :returns: The cluster's control plane ACL configuration. @@ -558,8 +552,6 @@ def control_plane_acl_update( """ Updates the ACL configuration for this cluster's control plane. - NOTE: Control Plane ACLs may not currently be available to all users. - API Documentation: https://techdocs.akamai.com/linode-api/reference/put-lke-cluster-acl :param acl: The ACL configuration to apply to this cluster. @@ -589,8 +581,6 @@ def control_plane_acl_delete(self): This has the same effect as calling control_plane_acl_update with the `enabled` field set to False. Access controls are disabled and all rules are deleted. - NOTE: Control Plane ACLs may not currently be available to all users. - API Documentation: https://techdocs.akamai.com/linode-api/reference/delete-lke-cluster-acl """ self._client.delete( From baee4dcc2cf4450efe2344701575c461a8769452 Mon Sep 17 00:00:00 2001 From: Lena Garber <114949949+lgarber-akamai@users.noreply.github.com> Date: Fri, 28 Feb 2025 09:21:45 -0500 Subject: [PATCH 07/10] Drop the LKE ACL `addresses` field when its value is null (#514) * Drop addresses key from ACL if None * Fix create * Add creation unit test * Account for LKE cluster ACL API change * make format --- linode_api4/groups/lke.py | 2 +- linode_api4/objects/lke.py | 3 +- test/integration/models/lke/test_lke.py | 21 ++++++++++++-- test/unit/objects/lke_test.py | 38 +++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/linode_api4/groups/lke.py b/linode_api4/groups/lke.py index d64d45536..4d13bb650 100644 --- a/linode_api4/groups/lke.py +++ b/linode_api4/groups/lke.py @@ -131,7 +131,7 @@ def cluster_create( result = self.client.post( "/lke/clusters", - data=_flatten_request_body_recursive(drop_null_keys(params)), + data=drop_null_keys(_flatten_request_body_recursive(params)), ) if "id" not in result: diff --git a/linode_api4/objects/lke.py b/linode_api4/objects/lke.py index 12d21f21f..2f670f2b9 100644 --- a/linode_api4/objects/lke.py +++ b/linode_api4/objects/lke.py @@ -14,6 +14,7 @@ Region, Type, ) +from linode_api4.util import drop_null_keys class LKEType(Base): @@ -566,7 +567,7 @@ def control_plane_acl_update( result = self._client.put( f"{LKECluster.api_endpoint}/control_plane_acl", model=self, - data={"acl": acl}, + data={"acl": drop_null_keys(acl)}, ) acl = result.get("acl") diff --git a/test/integration/models/lke/test_lke.py b/test/integration/models/lke/test_lke.py index 7355ca40b..794bc3203 100644 --- a/test/integration/models/lke/test_lke.py +++ b/test/integration/models/lke/test_lke.py @@ -45,7 +45,7 @@ def lke_cluster(test_linode_client): cluster.delete() -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def lke_cluster_with_acl(test_linode_client): node_type = test_linode_client.linode.types()[1] # g6-standard-1 version = test_linode_client.lke.versions()[0] @@ -288,9 +288,10 @@ def test_lke_cluster_acl(lke_cluster_with_acl): acl = cluster.control_plane_acl_update( LKEClusterControlPlaneACLOptions( + enabled=True, addresses=LKEClusterControlPlaneACLAddressesOptions( ipv4=["10.0.0.2/32"] - ) + ), ) ) @@ -298,6 +299,20 @@ def test_lke_cluster_acl(lke_cluster_with_acl): assert acl.addresses.ipv4 == ["10.0.0.2/32"] +def test_lke_cluster_update_acl_null_addresses(lke_cluster_with_acl): + cluster = lke_cluster_with_acl + + # Addresses should not be included in the request if it's null, + # else an error will be returned by the API. + # See: TPT-3489 + acl = cluster.control_plane_acl_update( + {"enabled": False, "addresses": None} + ) + + assert acl == cluster.control_plane_acl + assert acl.addresses.ipv4 == [] + + def test_lke_cluster_disable_acl(lke_cluster_with_acl): cluster = lke_cluster_with_acl @@ -311,7 +326,7 @@ def test_lke_cluster_disable_acl(lke_cluster_with_acl): assert acl.enabled is False assert acl == cluster.control_plane_acl - assert acl.addresses.ipv4 == ["10.0.0.2/32"] + assert acl.addresses.ipv4 == [] cluster.control_plane_acl_delete() diff --git a/test/unit/objects/lke_test.py b/test/unit/objects/lke_test.py index 1a39b69bc..1f397afac 100644 --- a/test/unit/objects/lke_test.py +++ b/test/unit/objects/lke_test.py @@ -498,3 +498,41 @@ def test_populate_with_mixed_types(self): assert self.pool.nodes[0].id == "node7" assert self.pool.nodes[1].id == "node8" assert self.pool.nodes[2].id == "node9" + + def test_cluster_create_acl_null_addresses(self): + with self.mock_post("lke/clusters") as m: + self.client.lke.cluster_create( + region="us-mia", + label="foobar", + kube_version="1.32", + node_pools=[self.client.lke.node_pool("g6-standard-1", 3)], + control_plane={ + "acl": { + "enabled": False, + "addresses": None, + } + }, + ) + + # Addresses should not be included in the API request if it's null + # See: TPT-3489 + assert m.call_data["control_plane"] == { + "acl": { + "enabled": False, + } + } + + def test_cluster_update_acl_null_addresses(self): + cluster = LKECluster(self.client, 18881) + + with self.mock_put("lke/clusters/18881/control_plane_acl") as m: + cluster.control_plane_acl_update( + { + "enabled": True, + "addresses": None, + } + ) + + # Addresses should not be included in the API request if it's null + # See: TPT-3489 + assert m.call_data == {"acl": {"enabled": True}} From 6cf8071322cd8d0d526486f863b11e058e79f3b0 Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-akamai@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:51:41 -0800 Subject: [PATCH 08/10] Improve slack test notifications (#516) * gha test 1 * update submodule * add test summary * gha test 2 * gha test 3 * gha test 4 * gha test 5 * gha test 6 * gha test 7 * gha test 8 * gha test 9 * gha test 10 * gha test 11 * gha test 12 * gha test 13 * gha test 13 * gha test 14 * gha test 15 * gha test 16 * gha test 17 * gha test 18 * improve slack test notification --- .github/workflows/e2e-test.yml | 32 ++++++++++++++++++++++++++++---- e2e_scripts | 2 +- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 229aba540..c0ccc8e87 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -173,6 +173,8 @@ jobs: runs-on: ubuntu-latest needs: [integration-tests] if: always() && github.repository == 'linode/linode_api4-python' # Run even if integration tests fail and only on main repository + outputs: + summary: ${{ steps.set-test-summary.outputs.summary }} steps: - name: Checkout code @@ -197,7 +199,6 @@ jobs: - name: Set release version env run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Add variables and upload test results if: always() run: | @@ -213,12 +214,24 @@ jobs: LINODE_CLI_OBJ_ACCESS_KEY: ${{ secrets.LINODE_CLI_OBJ_ACCESS_KEY }} LINODE_CLI_OBJ_SECRET_KEY: ${{ secrets.LINODE_CLI_OBJ_SECRET_KEY }} + - name: Generate test summary and save to output + id: set-test-summary + run: | + filename=$(ls | grep -E '^[0-9]{12}_sdk_test_report\.xml$') + test_output=$(python3 e2e_scripts/tod_scripts/generate_test_summary.py "${filename}") + { + echo 'summary<> "$GITHUB_OUTPUT" + notify-slack: runs-on: ubuntu-latest - needs: [integration-tests] + needs: [integration-tests, process-upload-report] if: ${{ (success() || failure()) }} # Run even if integration tests fail and only on main repository steps: - name: Notify Slack + id: main_message uses: slackapi/slack-github-action@v2.0.0 with: method: chat.postMessage @@ -229,7 +242,7 @@ jobs: - type: section text: type: mrkdwn - text: ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* :white_check_mark:" + text: ":rocket: *${{ github.workflow }} Completed in: ${{ github.repository }}* ${{ needs.integration-tests.result == 'success' && ':white_check_mark:' || ':failed:' }}" - type: divider - type: section fields: @@ -247,4 +260,15 @@ jobs: - type: context elements: - type: mrkdwn - text: "Triggered by: :bust_in_silhouette: `${{ github.actor }}`" \ No newline at end of file + text: "Triggered by: :bust_in_silhouette: `${{ github.actor }}`" + + - name: Test summary thread + if: success() + uses: slackapi/slack-github-action@v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: ${{ secrets.SLACK_CHANNEL_ID }} + thread_ts: "${{ steps.main_message.outputs.ts }}" + text: "${{ needs.process-upload-report.outputs.summary }}" diff --git a/e2e_scripts b/e2e_scripts index 0f2ff0169..5d0054351 160000 --- a/e2e_scripts +++ b/e2e_scripts @@ -1 +1 @@ -Subproject commit 0f2ff016956090c6fff046f4479e7efe8d0086e5 +Subproject commit 5d0054351277faa68a224172674210795cb36646 From a24d29738ca326638669bcdde4c5db5604265f37 Mon Sep 17 00:00:00 2001 From: Zhiwei Liang <121905282+zliang-akamai@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:40:37 -0500 Subject: [PATCH 09/10] Add missed filters to supported endpoints (#508) --- linode_api4/groups/account.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/linode_api4/groups/account.py b/linode_api4/groups/account.py index c2c69c624..564e55eea 100644 --- a/linode_api4/groups/account.py +++ b/linode_api4/groups/account.py @@ -101,7 +101,7 @@ def settings(self): s = AccountSettings(self.client, result["managed"], result) return s - def invoices(self): + def invoices(self, *filters): """ Returns Invoices issued to this account. @@ -112,9 +112,9 @@ def invoices(self): :returns: Invoices issued to this account. :rtype: PaginatedList of Invoice """ - return self.client._get_and_filter(Invoice) + return self.client._get_and_filter(Invoice, *filters) - def payments(self): + def payments(self, *filters): """ Returns a list of Payments made on this account. @@ -123,7 +123,7 @@ def payments(self): :returns: A list of payments made on this account. :rtype: PaginatedList of Payment """ - return self.client._get_and_filter(Payment) + return self.client._get_and_filter(Payment, *filters) def oauth_clients(self, *filters): """ @@ -337,7 +337,7 @@ def add_promo_code(self, promo_code): json=resp, ) - def service_transfers(self): + def service_transfers(self, *filters): """ Returns a collection of all created and accepted Service Transfers for this account, regardless of the user that created or accepted the transfer. @@ -347,7 +347,7 @@ def service_transfers(self): :rtype: PaginatedList of ServiceTransfer """ - return self.client._get_and_filter(ServiceTransfer) + return self.client._get_and_filter(ServiceTransfer, *filters) def service_transfer_create(self, entities): """ From f424bd212ac55e1ae0540a485be91433441abb17 Mon Sep 17 00:00:00 2001 From: Youjung Kim <126618609+ykim-akamai@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:02:18 -0800 Subject: [PATCH 10/10] update submodule (#517) --- e2e_scripts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_scripts b/e2e_scripts index 5d0054351..3265074d0 160000 --- a/e2e_scripts +++ b/e2e_scripts @@ -1 +1 @@ -Subproject commit 5d0054351277faa68a224172674210795cb36646 +Subproject commit 3265074d0d7ff8db6ce5207084051e1fc45d0763