From d795688d06d9e5264dc1acdcf3c881eda24ebcd6 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Tue, 16 Dec 2025 11:45:09 -0500 Subject: [PATCH 01/30] =?UTF-8?q?=F0=9F=99=88=20chore:=20add=20examples/co?= =?UTF-8?q?nfig.yml=20to=20.gitignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ignore local example configuration file to prevent accidental commits of user-specific settings. 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7587e84..ab48a64 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ Thumbs.db *.local .env.local deepfreeze-blog-post.md +examples/config.yml From bc70516d097140490ee5d4b77304a41c7f941c7a Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Fri, 16 Jan 2026 09:57:07 -0500 Subject: [PATCH 02/30] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20Azure=20Blob=20?= =?UTF-8?q?Storage=20support=20and=20refactor=20S3=20client?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Separate AWS implementation into dedicated aws_client.py - Add AzureBlobClient implementing S3Client interface for Azure Blob Storage - Update s3client.py to contain only abstract base class and factory - Add azure-storage-blob as optional dependency [azure] - Map S3 concepts to Azure equivalents (buckets→containers, Glacier→Archive tier) --- .../deepfreeze_core/__init__.py | 7 + .../deepfreeze_core/aws_client.py | 450 +++++++++++++++ .../deepfreeze_core/azure_client.py | 537 ++++++++++++++++++ .../deepfreeze_core/constants.py | 2 +- .../deepfreeze_core/s3client.py | 460 +-------------- packages/deepfreeze-core/pyproject.toml | 4 + 6 files changed, 1012 insertions(+), 448 deletions(-) create mode 100644 packages/deepfreeze-core/deepfreeze_core/aws_client.py create mode 100644 packages/deepfreeze-core/deepfreeze_core/azure_client.py diff --git a/packages/deepfreeze-core/deepfreeze_core/__init__.py b/packages/deepfreeze-core/deepfreeze_core/__init__.py index 38d63ac..8f2f28f 100644 --- a/packages/deepfreeze-core/deepfreeze_core/__init__.py +++ b/packages/deepfreeze-core/deepfreeze_core/__init__.py @@ -62,6 +62,12 @@ s3_client_factory, ) +# Conditional Azure export (azure-storage-blob is optional) +try: + from deepfreeze_core.azure_client import AzureBlobClient +except ImportError: + AzureBlobClient = None # type: ignore[misc,assignment] + # Export commonly used utilities from deepfreeze_core.utilities import ( check_restore_status, @@ -112,6 +118,7 @@ "Settings", # S3 Client "AwsS3Client", + "AzureBlobClient", "S3Client", "s3_client_factory", # ES Client diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py new file mode 100644 index 0000000..9f3f6a1 --- /dev/null +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -0,0 +1,450 @@ +""" +aws_client.py + +AWS S3 client implementation for the deepfreeze package. +""" + +import logging + +import boto3 +from botocore.exceptions import ClientError + +from deepfreeze_core.exceptions import ActionError +from deepfreeze_core.s3client import S3Client + + +class AwsS3Client(S3Client): + """ + An S3 client object for use with AWS. + """ + + def __init__(self) -> None: + self.loggit = logging.getLogger("deepfreeze.s3client") + try: + self.client = boto3.client("s3") + # Validate credentials by attempting a simple operation + self.loggit.debug("Validating AWS credentials") + self.client.list_buckets() + self.loggit.info("AWS S3 Client initialized successfully") + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + self.loggit.error( + "Failed to initialize AWS S3 Client: %s - %s", error_code, e + ) + if error_code in ["InvalidAccessKeyId", "SignatureDoesNotMatch"]: + raise ActionError( + "AWS credentials are invalid or not configured. " + "Check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY." + ) from e + elif error_code == "AccessDenied": + raise ActionError( + "AWS credentials do not have sufficient permissions. " + "Minimum required: s3:ListAllMyBuckets" + ) from e + raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e + except Exception as e: + self.loggit.error( + "Failed to initialize AWS S3 Client: %s", e, exc_info=True + ) + raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e + + def test_connection(self) -> bool: + """ + Test S3 connection and validate credentials. + + :return: True if credentials are valid and S3 is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing S3 connection") + self.client.list_buckets() + return True + except ClientError as e: + self.loggit.error("S3 connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + self.loggit.info(f"Creating bucket: {bucket_name}") + if self.bucket_exists(bucket_name): + self.loggit.info(f"Bucket {bucket_name} already exists") + raise ActionError(f"Bucket {bucket_name} already exists") + try: + # Add region handling for bucket creation + # Get the region from the client configuration + region = self.client.meta.region_name + self.loggit.debug(f"Creating bucket in region: {region}") + + # AWS requires LocationConstraint for all regions except us-east-1 + if region and region != "us-east-1": + self.client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={"LocationConstraint": region}, + ) + self.loggit.info( + f"Successfully created bucket {bucket_name} in region {region}" + ) + else: + self.client.create_bucket(Bucket=bucket_name) + self.loggit.info( + f"Successfully created bucket {bucket_name} in us-east-1" + ) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + self.loggit.error( + f"Error creating bucket {bucket_name}: {error_code} - {e}" + ) + raise ActionError(f"Error creating bucket {bucket_name}: {e}") from e + + def bucket_exists(self, bucket_name: str) -> bool: + self.loggit.debug(f"Checking if bucket {bucket_name} exists") + try: + self.client.head_bucket(Bucket=bucket_name) + self.loggit.debug(f"Bucket {bucket_name} exists") + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + self.loggit.debug(f"Bucket {bucket_name} does not exist") + return False + else: + self.loggit.error( + "Error checking bucket existence for %s: %s", bucket_name, e + ) + raise ActionError(e) from e + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[dict], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Restores objects from Glacier storage class back to an instant access tier. + + Args: + bucket_name (str): The name of the bucket + base_path (str): The base path (prefix) of the objects to thaw + object_keys (list[dict]): A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.) + restore_days (int): The number of days to keep the object restored + retrieval_tier (str): The retrieval tier to use + + Returns: + None + """ + self.loggit.info( + "Starting thaw operation - bucket: %s, base_path: %s, objects: %d, restore_days: %d, tier: %s", + bucket_name, + base_path, + len(object_keys), + restore_days, + retrieval_tier, + ) + + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, obj in enumerate(object_keys, 1): + # Extract key from object metadata dict + key = obj.get("Key") if isinstance(obj, dict) else obj + + if not key.startswith(base_path): + skipped_count += 1 + continue # Skip objects outside the base path + + # Get storage class from object metadata (if available) or fetch it + if isinstance(obj, dict) and "StorageClass" in obj: + storage_class = obj.get("StorageClass", "") + else: + try: + response = self.client.head_object(Bucket=bucket_name, Key=key) + storage_class = response.get("StorageClass", "") + except Exception as e: + error_count += 1 + self.loggit.error( + "Error getting metadata for object %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + continue + + try: + if storage_class in ["GLACIER", "DEEP_ARCHIVE", "GLACIER_IR"]: + self.loggit.debug( + "Restoring object %d/%d: %s from %s", + idx, + len(object_keys), + key, + storage_class, + ) + self.client.restore_object( + Bucket=bucket_name, + Key=key, + RestoreRequest={ + "Days": restore_days, + "GlacierJobParameters": {"Tier": retrieval_tier}, + }, + ) + restored_count += 1 + else: + self.loggit.debug( + "Skipping object %d/%d: %s (storage class: %s, not in Glacier)", + idx, + len(object_keys), + key, + storage_class, + ) + skipped_count += 1 + + except Exception as e: + error_count += 1 + self.loggit.error( + "Error restoring object %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + + # Log summary + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys), + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Moves objects back to a Glacier-tier storage class. + + Args: + bucket_name (str): The name of the bucket + path (str): The path to the objects to refreeze + storage_class (str): The storage class to move the objects to + + Returns: + None + """ + self.loggit.info( + "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class, + ) + + refrozen_count = 0 + error_count = 0 + + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=path) + + for page_num, page in enumerate(pages, 1): + if "Contents" in page: + page_objects = len(page["Contents"]) + self.loggit.debug( + "Processing page %d with %d objects", page_num, page_objects + ) + + for obj_num, obj in enumerate(page["Contents"], 1): + key = obj["Key"] + current_storage = obj.get("StorageClass", "STANDARD") + + try: + # Copy the object with a new storage class + self.loggit.debug( + "Refreezing object %d/%d in page %d: %s (from %s to %s)", + obj_num, + page_objects, + page_num, + key, + current_storage, + storage_class, + ) + self.client.copy_object( + Bucket=bucket_name, + CopySource={"Bucket": bucket_name, "Key": key}, + Key=key, + StorageClass=storage_class, + ) + refrozen_count += 1 + + except Exception as e: + error_count += 1 + self.loggit.error( + "Error refreezing object %s: %s (type: %s)", + key, + str(e), + type(e).__name__, + exc_info=True, + ) + + # Log summary + self.loggit.info( + "Refreeze operation completed - refrozen: %d, errors: %d", + refrozen_count, + error_count, + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: + """ + List objects in a bucket with a given prefix. + + Args: + bucket_name (str): The name of the bucket to list objects from. + prefix (str): The prefix to use when listing objects. + + Returns: + list[dict]: A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.). + """ + self.loggit.info( + f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" + ) + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + objects = [] + + for page in pages: + if "Contents" in page: + for obj in page["Contents"]: + objects.append(obj) + + return objects + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete a bucket with the given name. + + Args: + bucket_name (str): The name of the bucket to delete. + force (bool): If True, empty the bucket before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting bucket: {bucket_name}") + try: + # If force=True, empty the bucket first + if force: + self.loggit.info(f"Emptying bucket {bucket_name} before deletion") + try: + # List and delete all objects + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name) + + for page in pages: + if "Contents" in page: + objects = [{"Key": obj["Key"]} for obj in page["Contents"]] + if objects: + self.client.delete_objects( + Bucket=bucket_name, Delete={"Objects": objects} + ) + self.loggit.debug( + f"Deleted {len(objects)} objects from {bucket_name}" + ) + except ClientError as e: + if e.response["Error"]["Code"] != "NoSuchBucket": + self.loggit.warning(f"Error emptying bucket {bucket_name}: {e}") + + self.client.delete_bucket(Bucket=bucket_name) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Put an object in a bucket. + + Args: + bucket_name (str): The name of the bucket to put the object in. + key (str): The key of the object to put. + body (str): The body of the object to put. + + Returns: + None + """ + self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") + try: + self.client.put_object(Bucket=bucket_name, Key=key, Body=body) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all buckets. + + Returns: + list[str]: A list of bucket names. + """ + self.loggit.info("Listing buckets") + try: + response = self.client.list_buckets() + buckets = response.get("Buckets", []) + bucket_names = [bucket["Name"] for bucket in buckets] + if prefix: + bucket_names = [ + name for name in bucket_names if name.startswith(prefix) + ] + return bucket_names + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for an object without downloading it. + + Args: + bucket_name (str): The name of the bucket. + key (str): The object key. + + Returns: + dict: Object metadata including Restore status if applicable. + """ + self.loggit.debug(f"Getting metadata for s3://{bucket_name}/{key}") + try: + response = self.client.head_object(Bucket=bucket_name, Key=key) + return response + except ClientError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") from e + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy an object from one bucket to another. + + Args: + Bucket (str): The name of the destination bucket. + Key (str): The key for the copied object. + CopySource (dict[str, str]): The source bucket and key. + StorageClass (str): The storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying object {Key} to bucket {Bucket}") + try: + self.client.copy_object( + Bucket=Bucket, + CopySource=CopySource, + Key=Key, + StorageClass=StorageClass, + ) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py new file mode 100644 index 0000000..c396f11 --- /dev/null +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -0,0 +1,537 @@ +""" +azure_client.py + +Azure Blob Storage client implementation for the deepfreeze package. +Maps S3 concepts to Azure equivalents: +- S3 Bucket -> Azure Blob Container +- S3 Object -> Azure Blob +- Glacier/Deep Archive -> Azure Archive tier +- S3 restore -> Azure blob rehydration +""" + +import logging +import os + +from azure.core.exceptions import ( + AzureError, + ResourceExistsError, + ResourceNotFoundError, +) +from azure.storage.blob import BlobServiceClient, StandardBlobTier + +from deepfreeze_core.exceptions import ActionError +from deepfreeze_core.s3client import S3Client + + +class AzureBlobClient(S3Client): + """ + Azure Blob Storage client implementing the S3Client interface. + """ + + def __init__(self) -> None: + self.loggit = logging.getLogger("deepfreeze.azure_client") + try: + # Azure SDK uses connection string from environment variable + # AZURE_STORAGE_CONNECTION_STRING or account name + key + connection_string = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + if connection_string: + self.service_client = BlobServiceClient.from_connection_string( + connection_string + ) + self.loggit.debug("Using AZURE_STORAGE_CONNECTION_STRING for auth") + else: + # Alternative: account name + key + account_name = os.environ.get("AZURE_STORAGE_ACCOUNT") + account_key = os.environ.get("AZURE_STORAGE_KEY") + if account_name and account_key: + account_url = f"https://{account_name}.blob.core.windows.net" + self.service_client = BlobServiceClient( + account_url=account_url, credential=account_key + ) + self.loggit.debug( + "Using AZURE_STORAGE_ACCOUNT + AZURE_STORAGE_KEY for auth" + ) + else: + raise ActionError( + "Azure credentials not configured. Set AZURE_STORAGE_CONNECTION_STRING " + "or both AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY." + ) + + # Validate credentials + self.loggit.debug("Validating Azure credentials") + list(self.service_client.list_containers(max_results=1)) + self.loggit.info("Azure Blob Storage Client initialized successfully") + + except AzureError as e: + self.loggit.error("Failed to initialize Azure Blob Storage Client: %s", e) + raise ActionError( + f"Failed to initialize Azure Blob Storage Client: {e}" + ) from e + except Exception as e: + self.loggit.error( + "Failed to initialize Azure Blob Storage Client: %s", e, exc_info=True + ) + raise ActionError( + f"Failed to initialize Azure Blob Storage Client: {e}" + ) from e + + def test_connection(self) -> bool: + """ + Test Azure connection and validate credentials. + + :return: True if credentials are valid and Azure is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing Azure connection") + list(self.service_client.list_containers(max_results=1)) + return True + except AzureError as e: + self.loggit.error("Azure connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + """Create an Azure Blob container (equivalent to S3 bucket).""" + self.loggit.info(f"Creating container: {bucket_name}") + if self.bucket_exists(bucket_name): + self.loggit.info(f"Container {bucket_name} already exists") + raise ActionError(f"Container {bucket_name} already exists") + try: + self.service_client.create_container(bucket_name) + self.loggit.info(f"Successfully created container {bucket_name}") + except ResourceExistsError as e: + raise ActionError(f"Container {bucket_name} already exists") from e + except AzureError as e: + self.loggit.error(f"Error creating container {bucket_name}: {e}") + raise ActionError(f"Error creating container {bucket_name}: {e}") from e + + def bucket_exists(self, bucket_name: str) -> bool: + """Check if an Azure Blob container exists.""" + self.loggit.debug(f"Checking if container {bucket_name} exists") + try: + container_client = self.service_client.get_container_client(bucket_name) + container_client.get_container_properties() + self.loggit.debug(f"Container {bucket_name} exists") + return True + except ResourceNotFoundError: + self.loggit.debug(f"Container {bucket_name} does not exist") + return False + except AzureError as e: + self.loggit.error( + "Error checking container existence for %s: %s", bucket_name, e + ) + raise ActionError(e) from e + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[dict], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Rehydrate blobs from Archive tier. + + Azure rehydration is different from S3 Glacier restore: + - Uses set_standard_blob_tier() with rehydrate_priority + - Tier options: 'Hot', 'Cool' (Archive cannot be read directly) + - Rehydrate priorities: 'Standard' (up to 15 hours), 'High' (under 1 hour) + - Note: restore_days is not used in Azure - rehydration changes the tier permanently + + Args: + bucket_name (str): The name of the container + base_path (str): The base path (prefix) of the blobs to thaw + object_keys (list[dict]): A list of blob metadata dictionaries + restore_days (int): Not used in Azure (kept for interface compatibility) + retrieval_tier (str): 'Standard', 'Bulk', or 'Expedited' (mapped to Azure priorities) + + Returns: + None + """ + self.loggit.info( + "Starting thaw operation - container: %s, base_path: %s, objects: %d, tier: %s", + bucket_name, + base_path, + len(object_keys), + retrieval_tier, + ) + + # Map S3 retrieval tier to Azure rehydrate priority + priority_map = { + "Standard": "Standard", + "Bulk": "Standard", # Azure doesn't have Bulk, use Standard + "Expedited": "High", + } + rehydrate_priority = priority_map.get(retrieval_tier, "Standard") + + container_client = self.service_client.get_container_client(bucket_name) + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, obj in enumerate(object_keys, 1): + key = obj.get("Key") if isinstance(obj, dict) else obj + + if not key.startswith(base_path): + skipped_count += 1 + continue + + try: + blob_client = container_client.get_blob_client(key) + properties = blob_client.get_blob_properties() + current_tier = properties.blob_tier + + if current_tier == "Archive": + self.loggit.debug( + "Rehydrating blob %d/%d: %s from Archive (priority: %s)", + idx, + len(object_keys), + key, + rehydrate_priority, + ) + # Rehydrate to Hot tier + blob_client.set_standard_blob_tier( + StandardBlobTier.HOT, rehydrate_priority=rehydrate_priority + ) + restored_count += 1 + else: + self.loggit.debug( + "Skipping blob %d/%d: %s (tier: %s, not Archive)", + idx, + len(object_keys), + key, + current_tier, + ) + skipped_count += 1 + + except AzureError as e: + error_count += 1 + self.loggit.error( + "Error rehydrating blob %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys), + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Move blobs back to Archive tier. + + Maps S3 storage classes to Azure tiers: + - GLACIER -> Archive + - DEEP_ARCHIVE -> Archive (Azure has only one archive tier) + - GLACIER_IR -> Cool (closest equivalent) + + Args: + bucket_name (str): The name of the container + path (str): The path (prefix) to the blobs to refreeze + storage_class (str): The S3-style storage class to move blobs to + + Returns: + None + """ + self.loggit.info( + "Starting refreeze operation - container: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class, + ) + + # Map S3 storage class to Azure tier + tier_map = { + "GLACIER": StandardBlobTier.ARCHIVE, + "DEEP_ARCHIVE": StandardBlobTier.ARCHIVE, + "GLACIER_IR": StandardBlobTier.COOL, + } + target_tier = tier_map.get(storage_class, StandardBlobTier.ARCHIVE) + + container_client = self.service_client.get_container_client(bucket_name) + refrozen_count = 0 + error_count = 0 + + # List blobs with prefix + blobs = container_client.list_blobs(name_starts_with=path) + + for blob in blobs: + try: + blob_client = container_client.get_blob_client(blob.name) + current_tier = blob.blob_tier + self.loggit.debug( + "Refreezing blob: %s (from %s to %s)", + blob.name, + current_tier, + target_tier, + ) + blob_client.set_standard_blob_tier(target_tier) + refrozen_count += 1 + except AzureError as e: + error_count += 1 + self.loggit.error( + "Error refreezing blob %s: %s (type: %s)", + blob.name, + str(e), + type(e).__name__, + exc_info=True, + ) + + self.loggit.info( + "Refreeze operation completed - refrozen: %d, errors: %d", + refrozen_count, + error_count, + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: + """ + List blobs in a container with a given prefix. + + Args: + bucket_name (str): The name of the container to list blobs from. + prefix (str): The prefix to use when listing blobs. + + Returns: + list[dict]: A list of blob metadata dictionaries with S3-compatible keys. + """ + self.loggit.info( + f"Listing blobs in container: {bucket_name} with prefix: {prefix}" + ) + + container_client = self.service_client.get_container_client(bucket_name) + objects = [] + + try: + blobs = container_client.list_blobs(name_starts_with=prefix) + for blob in blobs: + # Map Azure properties to S3-like structure for compatibility + objects.append( + { + "Key": blob.name, + "Size": blob.size, + "LastModified": blob.last_modified, + "StorageClass": self._map_azure_tier_to_s3(blob.blob_tier), + # Azure-specific + "BlobTier": blob.blob_tier, + "ArchiveStatus": getattr(blob, "archive_status", None), + } + ) + return objects + except AzureError as e: + self.loggit.error("Error listing blobs: %s", e) + raise ActionError(f"Error listing blobs: {e}") from e + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete an Azure Blob container. + + Args: + bucket_name (str): The name of the container to delete. + force (bool): If True, empty the container before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting container: {bucket_name}") + try: + container_client = self.service_client.get_container_client(bucket_name) + + if force: + self.loggit.info(f"Emptying container {bucket_name} before deletion") + blobs = container_client.list_blobs() + for blob in blobs: + container_client.delete_blob(blob.name) + self.loggit.debug(f"Deleted blob {blob.name}") + + container_client.delete_container() + self.loggit.info(f"Container {bucket_name} deleted successfully") + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Upload a blob to a container. + + Args: + bucket_name (str): The name of the container to put the blob in. + key (str): The key of the blob to put. + body (str): The body of the blob to put. + + Returns: + None + """ + self.loggit.info(f"Putting blob: {key} in container: {bucket_name}") + try: + container_client = self.service_client.get_container_client(bucket_name) + blob_client = container_client.get_blob_client(key) + blob_client.upload_blob(body, overwrite=True) + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all containers. + + Args: + prefix (str): Optional prefix to filter container names. + + Returns: + list[str]: A list of container names. + """ + self.loggit.info("Listing containers") + try: + containers = self.service_client.list_containers() + container_names = [c.name for c in containers] + if prefix: + container_names = [ + name for name in container_names if name.startswith(prefix) + ] + return container_names + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for a blob without downloading it. + + Args: + bucket_name (str): The name of the container. + key (str): The blob key. + + Returns: + dict: Blob metadata including Restore status if applicable. + """ + self.loggit.debug(f"Getting metadata for azure://{bucket_name}/{key}") + try: + container_client = self.service_client.get_container_client(bucket_name) + blob_client = container_client.get_blob_client(key) + properties = blob_client.get_blob_properties() + + # Map to S3-like response structure + response = { + "ContentLength": properties.size, + "ContentType": properties.content_settings.content_type, + "LastModified": properties.last_modified, + "ETag": properties.etag, + "StorageClass": self._map_azure_tier_to_s3(properties.blob_tier), + # Azure rehydration status maps to S3 Restore header + "Restore": self._format_restore_header(properties), + # Azure-specific + "BlobTier": properties.blob_tier, + "ArchiveStatus": getattr(properties, "archive_status", None), + } + return response + except ResourceNotFoundError as e: + self.loggit.error(f"Blob not found: {key}") + raise ActionError(f"Blob not found: {key}") from e + except AzureError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") from e + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy a blob with storage class change. + + Args: + Bucket (str): The name of the destination container. + Key (str): The key for the copied blob. + CopySource (dict[str, str]): The source container and key. + StorageClass (str): The S3-style storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying blob {Key} to container {Bucket}") + try: + source_container = CopySource["Bucket"] + source_key = CopySource["Key"] + + # Get source URL + source_container_client = self.service_client.get_container_client( + source_container + ) + source_blob_client = source_container_client.get_blob_client(source_key) + source_url = source_blob_client.url + + # Copy to destination + dest_container_client = self.service_client.get_container_client(Bucket) + dest_blob_client = dest_container_client.get_blob_client(Key) + + # Start copy operation + dest_blob_client.start_copy_from_url(source_url) + + # Set storage tier + tier_map = { + "GLACIER": StandardBlobTier.ARCHIVE, + "DEEP_ARCHIVE": StandardBlobTier.ARCHIVE, + "STANDARD": StandardBlobTier.HOT, + "STANDARD_IA": StandardBlobTier.COOL, + } + target_tier = tier_map.get(StorageClass, StandardBlobTier.ARCHIVE) + dest_blob_client.set_standard_blob_tier(target_tier) + + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def _map_azure_tier_to_s3(self, azure_tier: str) -> str: + """ + Map Azure blob tier to S3 storage class for compatibility. + + Args: + azure_tier (str): The Azure blob tier. + + Returns: + str: The equivalent S3 storage class. + """ + tier_map = { + "Hot": "STANDARD", + "Cool": "STANDARD_IA", + "Cold": "ONEZONE_IA", + "Archive": "GLACIER", + } + return tier_map.get(azure_tier, "STANDARD") + + def _format_restore_header(self, properties) -> str: + """ + Format Azure rehydration status to S3-like Restore header. + + Args: + properties: The blob properties object. + + Returns: + str: S3-style restore header or None. + """ + archive_status = getattr(properties, "archive_status", None) + blob_tier = properties.blob_tier + + if blob_tier == "Archive": + if archive_status in [ + "rehydrate-pending-to-hot", + "rehydrate-pending-to-cool", + ]: + return 'ongoing-request="true"' + return None # Not restored + elif blob_tier in ("Hot", "Cool") and archive_status: + # Was rehydrated from archive + return 'ongoing-request="false"' + return None diff --git a/packages/deepfreeze-core/deepfreeze_core/constants.py b/packages/deepfreeze-core/deepfreeze_core/constants.py index b8f4510..784506c 100644 --- a/packages/deepfreeze-core/deepfreeze_core/constants.py +++ b/packages/deepfreeze-core/deepfreeze_core/constants.py @@ -9,7 +9,7 @@ SETTINGS_ID = "1" # Supported cloud providers -PROVIDERS = ["aws"] +PROVIDERS = ["aws", "azure"] # Repository thaw lifecycle states THAW_STATE_ACTIVE = "active" # Active repository, never been through thaw lifecycle diff --git a/packages/deepfreeze-core/deepfreeze_core/s3client.py b/packages/deepfreeze-core/deepfreeze_core/s3client.py index d8a55d2..24ed53c 100644 --- a/packages/deepfreeze-core/deepfreeze_core/s3client.py +++ b/packages/deepfreeze-core/deepfreeze_core/s3client.py @@ -6,12 +6,6 @@ """ import abc -import logging - -import boto3 -from botocore.exceptions import ClientError - -from deepfreeze_core.exceptions import ActionError class S3Client(metaclass=abc.ABCMeta): @@ -186,443 +180,6 @@ def copy_object( return -class AwsS3Client(S3Client): - """ - An S3 client object for use with AWS. - """ - - def __init__(self) -> None: - self.loggit = logging.getLogger("deepfreeze.s3client") - try: - self.client = boto3.client("s3") - # Validate credentials by attempting a simple operation - self.loggit.debug("Validating AWS credentials") - self.client.list_buckets() - self.loggit.info("AWS S3 Client initialized successfully") - except ClientError as e: - error_code = e.response.get("Error", {}).get("Code", "Unknown") - self.loggit.error( - "Failed to initialize AWS S3 Client: %s - %s", error_code, e - ) - if error_code in ["InvalidAccessKeyId", "SignatureDoesNotMatch"]: - raise ActionError( - "AWS credentials are invalid or not configured. " - "Check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY." - ) from e - elif error_code == "AccessDenied": - raise ActionError( - "AWS credentials do not have sufficient permissions. " - "Minimum required: s3:ListAllMyBuckets" - ) from e - raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e - except Exception as e: - self.loggit.error( - "Failed to initialize AWS S3 Client: %s", e, exc_info=True - ) - raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e - - def test_connection(self) -> bool: - """ - Test S3 connection and validate credentials. - - :return: True if credentials are valid and S3 is accessible - :rtype: bool - """ - try: - self.loggit.debug("Testing S3 connection") - self.client.list_buckets() - return True - except ClientError as e: - self.loggit.error("S3 connection test failed: %s", e) - return False - - def create_bucket(self, bucket_name: str) -> None: - self.loggit.info(f"Creating bucket: {bucket_name}") - if self.bucket_exists(bucket_name): - self.loggit.info(f"Bucket {bucket_name} already exists") - raise ActionError(f"Bucket {bucket_name} already exists") - try: - # Add region handling for bucket creation - # Get the region from the client configuration - region = self.client.meta.region_name - self.loggit.debug(f"Creating bucket in region: {region}") - - # AWS requires LocationConstraint for all regions except us-east-1 - if region and region != "us-east-1": - self.client.create_bucket( - Bucket=bucket_name, - CreateBucketConfiguration={"LocationConstraint": region}, - ) - self.loggit.info( - f"Successfully created bucket {bucket_name} in region {region}" - ) - else: - self.client.create_bucket(Bucket=bucket_name) - self.loggit.info( - f"Successfully created bucket {bucket_name} in us-east-1" - ) - except ClientError as e: - error_code = e.response.get("Error", {}).get("Code", "Unknown") - self.loggit.error( - f"Error creating bucket {bucket_name}: {error_code} - {e}" - ) - raise ActionError(f"Error creating bucket {bucket_name}: {e}") from e - - def bucket_exists(self, bucket_name: str) -> bool: - self.loggit.debug(f"Checking if bucket {bucket_name} exists") - try: - self.client.head_bucket(Bucket=bucket_name) - self.loggit.debug(f"Bucket {bucket_name} exists") - return True - except ClientError as e: - if e.response["Error"]["Code"] == "404": - self.loggit.debug(f"Bucket {bucket_name} does not exist") - return False - else: - self.loggit.error( - "Error checking bucket existence for %s: %s", bucket_name, e - ) - raise ActionError(e) from e - - def thaw( - self, - bucket_name: str, - base_path: str, - object_keys: list[dict], - restore_days: int = 7, - retrieval_tier: str = "Standard", - ) -> None: - """ - Restores objects from Glacier storage class back to an instant access tier. - - Args: - bucket_name (str): The name of the bucket - base_path (str): The base path (prefix) of the objects to thaw - object_keys (list[dict]): A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.) - restore_days (int): The number of days to keep the object restored - retrieval_tier (str): The retrieval tier to use - - Returns: - None - """ - self.loggit.info( - "Starting thaw operation - bucket: %s, base_path: %s, objects: %d, restore_days: %d, tier: %s", - bucket_name, - base_path, - len(object_keys), - restore_days, - retrieval_tier, - ) - - restored_count = 0 - skipped_count = 0 - error_count = 0 - - for idx, obj in enumerate(object_keys, 1): - # Extract key from object metadata dict - key = obj.get("Key") if isinstance(obj, dict) else obj - - if not key.startswith(base_path): - skipped_count += 1 - continue # Skip objects outside the base path - - # Get storage class from object metadata (if available) or fetch it - if isinstance(obj, dict) and "StorageClass" in obj: - storage_class = obj.get("StorageClass", "") - else: - try: - response = self.client.head_object(Bucket=bucket_name, Key=key) - storage_class = response.get("StorageClass", "") - except Exception as e: - error_count += 1 - self.loggit.error( - "Error getting metadata for object %d/%d (%s): %s (type: %s)", - idx, - len(object_keys), - key, - str(e), - type(e).__name__, - ) - continue - - try: - if storage_class in ["GLACIER", "DEEP_ARCHIVE", "GLACIER_IR"]: - self.loggit.debug( - "Restoring object %d/%d: %s from %s", - idx, - len(object_keys), - key, - storage_class, - ) - self.client.restore_object( - Bucket=bucket_name, - Key=key, - RestoreRequest={ - "Days": restore_days, - "GlacierJobParameters": {"Tier": retrieval_tier}, - }, - ) - restored_count += 1 - else: - self.loggit.debug( - "Skipping object %d/%d: %s (storage class: %s, not in Glacier)", - idx, - len(object_keys), - key, - storage_class, - ) - skipped_count += 1 - - except Exception as e: - error_count += 1 - self.loggit.error( - "Error restoring object %d/%d (%s): %s (type: %s)", - idx, - len(object_keys), - key, - str(e), - type(e).__name__, - ) - - # Log summary - self.loggit.info( - "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", - restored_count, - skipped_count, - error_count, - len(object_keys), - ) - - def refreeze( - self, bucket_name: str, path: str, storage_class: str = "GLACIER" - ) -> None: - """ - Moves objects back to a Glacier-tier storage class. - - Args: - bucket_name (str): The name of the bucket - path (str): The path to the objects to refreeze - storage_class (str): The storage class to move the objects to - - Returns: - None - """ - self.loggit.info( - "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", - bucket_name, - path, - storage_class, - ) - - refrozen_count = 0 - error_count = 0 - - paginator = self.client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name, Prefix=path) - - for page_num, page in enumerate(pages, 1): - if "Contents" in page: - page_objects = len(page["Contents"]) - self.loggit.debug( - "Processing page %d with %d objects", page_num, page_objects - ) - - for obj_num, obj in enumerate(page["Contents"], 1): - key = obj["Key"] - current_storage = obj.get("StorageClass", "STANDARD") - - try: - # Copy the object with a new storage class - self.loggit.debug( - "Refreezing object %d/%d in page %d: %s (from %s to %s)", - obj_num, - page_objects, - page_num, - key, - current_storage, - storage_class, - ) - self.client.copy_object( - Bucket=bucket_name, - CopySource={"Bucket": bucket_name, "Key": key}, - Key=key, - StorageClass=storage_class, - ) - refrozen_count += 1 - - except Exception as e: - error_count += 1 - self.loggit.error( - "Error refreezing object %s: %s (type: %s)", - key, - str(e), - type(e).__name__, - exc_info=True, - ) - - # Log summary - self.loggit.info( - "Refreeze operation completed - refrozen: %d, errors: %d", - refrozen_count, - error_count, - ) - - def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: - """ - List objects in a bucket with a given prefix. - - Args: - bucket_name (str): The name of the bucket to list objects from. - prefix (str): The prefix to use when listing objects. - - Returns: - list[dict]: A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.). - """ - self.loggit.info( - f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" - ) - paginator = self.client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) - objects = [] - - for page in pages: - if "Contents" in page: - for obj in page["Contents"]: - objects.append(obj) - - return objects - - def delete_bucket(self, bucket_name: str, force: bool = False) -> None: - """ - Delete a bucket with the given name. - - Args: - bucket_name (str): The name of the bucket to delete. - force (bool): If True, empty the bucket before deleting it. - - Returns: - None - """ - self.loggit.info(f"Deleting bucket: {bucket_name}") - try: - # If force=True, empty the bucket first - if force: - self.loggit.info(f"Emptying bucket {bucket_name} before deletion") - try: - # List and delete all objects - paginator = self.client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name) - - for page in pages: - if "Contents" in page: - objects = [{"Key": obj["Key"]} for obj in page["Contents"]] - if objects: - self.client.delete_objects( - Bucket=bucket_name, Delete={"Objects": objects} - ) - self.loggit.debug( - f"Deleted {len(objects)} objects from {bucket_name}" - ) - except ClientError as e: - if e.response["Error"]["Code"] != "NoSuchBucket": - self.loggit.warning(f"Error emptying bucket {bucket_name}: {e}") - - self.client.delete_bucket(Bucket=bucket_name) - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def put_object(self, bucket_name: str, key: str, body: str = "") -> None: - """ - Put an object in a bucket. - - Args: - bucket_name (str): The name of the bucket to put the object in. - key (str): The key of the object to put. - body (str): The body of the object to put. - - Returns: - None - """ - self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") - try: - self.client.put_object(Bucket=bucket_name, Key=key, Body=body) - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def list_buckets(self, prefix: str = None) -> list[str]: - """ - List all buckets. - - Returns: - list[str]: A list of bucket names. - """ - self.loggit.info("Listing buckets") - try: - response = self.client.list_buckets() - buckets = response.get("Buckets", []) - bucket_names = [bucket["Name"] for bucket in buckets] - if prefix: - bucket_names = [ - name for name in bucket_names if name.startswith(prefix) - ] - return bucket_names - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def head_object(self, bucket_name: str, key: str) -> dict: - """ - Retrieve metadata for an object without downloading it. - - Args: - bucket_name (str): The name of the bucket. - key (str): The object key. - - Returns: - dict: Object metadata including Restore status if applicable. - """ - self.loggit.debug(f"Getting metadata for s3://{bucket_name}/{key}") - try: - response = self.client.head_object(Bucket=bucket_name, Key=key) - return response - except ClientError as e: - self.loggit.error(f"Error getting metadata for {key}: {e}") - raise ActionError(f"Error getting metadata for {key}: {e}") from e - - def copy_object( - self, - Bucket: str, - Key: str, - CopySource: dict[str, str], - StorageClass: str = "GLACIER", - ) -> None: - """ - Copy an object from one bucket to another. - - Args: - Bucket (str): The name of the destination bucket. - Key (str): The key for the copied object. - CopySource (dict[str, str]): The source bucket and key. - StorageClass (str): The storage class to use. - - Returns: - None - """ - self.loggit.info(f"Copying object {Key} to bucket {Bucket}") - try: - self.client.copy_object( - Bucket=Bucket, - CopySource=CopySource, - Key=Key, - StorageClass=StorageClass, - ) - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def s3_client_factory(provider: str) -> S3Client: """ s3_client_factory method, returns an S3Client object implemented specific to @@ -630,7 +187,7 @@ def s3_client_factory(provider: str) -> S3Client: Args: provider (str): The provider to use for the S3Client object. Should - reference an implemented provider (aws, gcp, azure, etc) + reference an implemented provider (aws, azure, gcp, etc) Raises: NotImplementedError: raised if the provider is not implemented @@ -640,12 +197,21 @@ def s3_client_factory(provider: str) -> S3Client: S3Client: An S3Client object specific to the provider argument. """ if provider == "aws": + from deepfreeze_core.aws_client import AwsS3Client + return AwsS3Client() + elif provider == "azure": + from deepfreeze_core.azure_client import AzureBlobClient + + return AzureBlobClient() elif provider == "gcp": # Placeholder for GCP S3Client implementation raise NotImplementedError("GCP S3Client is not implemented yet") - elif provider == "azure": - # Placeholder for Azure S3Client implementation - raise NotImplementedError("Azure S3Client is not implemented yet") else: raise ValueError(f"Unsupported provider: {provider}") + + +# Backward-compatible re-exports +from deepfreeze_core.aws_client import AwsS3Client # noqa: E402, F401 + +__all__ = ["S3Client", "AwsS3Client", "s3_client_factory"] diff --git a/packages/deepfreeze-core/pyproject.toml b/packages/deepfreeze-core/pyproject.toml index 687de5c..aac5fad 100644 --- a/packages/deepfreeze-core/pyproject.toml +++ b/packages/deepfreeze-core/pyproject.toml @@ -42,6 +42,9 @@ dependencies = [ ] [project.optional-dependencies] +azure = [ + "azure-storage-blob>=12.0.0", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", @@ -49,6 +52,7 @@ dev = [ "black>=23.0.0", "ruff>=0.1.0", "moto>=4.0.0", + "azure-storage-blob>=12.0.0", ] [project.urls] From af4690815c58e5169461c74dc23771604b69a430 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Fri, 16 Jan 2026 10:26:42 -0500 Subject: [PATCH 03/30] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20Google=20Cloud?= =?UTF-8?q?=20Storage=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add GcpStorageClient implementing S3Client interface for GCS - Map S3 concepts to GCS equivalents (Glacier→Archive storage class) - Add google-cloud-storage as optional dependency [gcp] - Update factory to support "gcp" provider --- .../deepfreeze_core/__init__.py | 7 + .../deepfreeze_core/constants.py | 2 +- .../deepfreeze_core/gcp_client.py | 484 ++++++++++++++++++ .../deepfreeze_core/s3client.py | 5 +- packages/deepfreeze-core/pyproject.toml | 4 + 5 files changed, 499 insertions(+), 3 deletions(-) create mode 100644 packages/deepfreeze-core/deepfreeze_core/gcp_client.py diff --git a/packages/deepfreeze-core/deepfreeze_core/__init__.py b/packages/deepfreeze-core/deepfreeze_core/__init__.py index 8f2f28f..18d18f4 100644 --- a/packages/deepfreeze-core/deepfreeze_core/__init__.py +++ b/packages/deepfreeze-core/deepfreeze_core/__init__.py @@ -68,6 +68,12 @@ except ImportError: AzureBlobClient = None # type: ignore[misc,assignment] +# Conditional GCP export (google-cloud-storage is optional) +try: + from deepfreeze_core.gcp_client import GcpStorageClient +except ImportError: + GcpStorageClient = None # type: ignore[misc,assignment] + # Export commonly used utilities from deepfreeze_core.utilities import ( check_restore_status, @@ -119,6 +125,7 @@ # S3 Client "AwsS3Client", "AzureBlobClient", + "GcpStorageClient", "S3Client", "s3_client_factory", # ES Client diff --git a/packages/deepfreeze-core/deepfreeze_core/constants.py b/packages/deepfreeze-core/deepfreeze_core/constants.py index 784506c..248b5e5 100644 --- a/packages/deepfreeze-core/deepfreeze_core/constants.py +++ b/packages/deepfreeze-core/deepfreeze_core/constants.py @@ -9,7 +9,7 @@ SETTINGS_ID = "1" # Supported cloud providers -PROVIDERS = ["aws", "azure"] +PROVIDERS = ["aws", "azure", "gcp"] # Repository thaw lifecycle states THAW_STATE_ACTIVE = "active" # Active repository, never been through thaw lifecycle diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py new file mode 100644 index 0000000..bbd27b2 --- /dev/null +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -0,0 +1,484 @@ +""" +gcp_client.py + +Google Cloud Storage client implementation for the deepfreeze package. +Maps S3 concepts to GCP equivalents: +- S3 Bucket -> GCS Bucket +- S3 Object -> GCS Blob +- Glacier/Deep Archive -> GCS Archive storage class +- Storage classes: STANDARD, NEARLINE, COLDLINE, ARCHIVE +""" + +import logging +import os + +from google.api_core.exceptions import Conflict, GoogleAPIError, NotFound +from google.cloud import storage + +from deepfreeze_core.exceptions import ActionError +from deepfreeze_core.s3client import S3Client + + +class GcpStorageClient(S3Client): + """ + Google Cloud Storage client implementing the S3Client interface. + """ + + def __init__(self) -> None: + self.loggit = logging.getLogger("deepfreeze.gcp_client") + try: + # GCP SDK uses Application Default Credentials (ADC) + # Set GOOGLE_APPLICATION_CREDENTIALS env var to service account JSON path + # or use gcloud auth application-default login + project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") + if project_id: + self.client = storage.Client(project=project_id) + self.loggit.debug("Using GOOGLE_CLOUD_PROJECT: %s", project_id) + else: + self.client = storage.Client() + self.loggit.debug("Using default project from credentials") + + # Validate credentials by listing buckets (limited to 1) + self.loggit.debug("Validating GCP credentials") + list(self.client.list_buckets(max_results=1)) + self.loggit.info("GCP Storage Client initialized successfully") + + except GoogleAPIError as e: + self.loggit.error("Failed to initialize GCP Storage Client: %s", e) + raise ActionError( + f"Failed to initialize GCP Storage Client: {e}" + ) from e + except Exception as e: + self.loggit.error( + "Failed to initialize GCP Storage Client: %s", e, exc_info=True + ) + raise ActionError( + f"Failed to initialize GCP Storage Client: {e}" + ) from e + + def test_connection(self) -> bool: + """ + Test GCP connection and validate credentials. + + :return: True if credentials are valid and GCS is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing GCP connection") + list(self.client.list_buckets(max_results=1)) + return True + except GoogleAPIError as e: + self.loggit.error("GCP connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + """Create a GCS bucket.""" + self.loggit.info(f"Creating bucket: {bucket_name}") + if self.bucket_exists(bucket_name): + self.loggit.info(f"Bucket {bucket_name} already exists") + raise ActionError(f"Bucket {bucket_name} already exists") + try: + bucket = self.client.bucket(bucket_name) + # Use default location, can be customized via GOOGLE_CLOUD_LOCATION env var + location = os.environ.get("GOOGLE_CLOUD_LOCATION", "US") + self.client.create_bucket(bucket, location=location) + self.loggit.info( + f"Successfully created bucket {bucket_name} in location {location}" + ) + except Conflict as e: + raise ActionError(f"Bucket {bucket_name} already exists") from e + except GoogleAPIError as e: + self.loggit.error(f"Error creating bucket {bucket_name}: {e}") + raise ActionError(f"Error creating bucket {bucket_name}: {e}") from e + + def bucket_exists(self, bucket_name: str) -> bool: + """Check if a GCS bucket exists.""" + self.loggit.debug(f"Checking if bucket {bucket_name} exists") + try: + bucket = self.client.bucket(bucket_name) + bucket.reload() + self.loggit.debug(f"Bucket {bucket_name} exists") + return True + except NotFound: + self.loggit.debug(f"Bucket {bucket_name} does not exist") + return False + except GoogleAPIError as e: + self.loggit.error( + "Error checking bucket existence for %s: %s", bucket_name, e + ) + raise ActionError(e) from e + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[dict], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Move objects from Archive/Coldline to Standard storage class. + + Unlike AWS Glacier, GCS Archive objects are immediately accessible, + just with higher retrieval costs. "Thawing" means changing the + storage class to STANDARD for faster/cheaper access. + + Args: + bucket_name (str): The name of the bucket + base_path (str): The base path (prefix) of the objects to thaw + object_keys (list[dict]): A list of object metadata dictionaries + restore_days (int): Not used in GCP (kept for interface compatibility) + retrieval_tier (str): Not used in GCP (kept for interface compatibility) + + Returns: + None + """ + self.loggit.info( + "Starting thaw operation - bucket: %s, base_path: %s, objects: %d", + bucket_name, + base_path, + len(object_keys), + ) + + bucket = self.client.bucket(bucket_name) + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, obj in enumerate(object_keys, 1): + key = obj.get("Key") if isinstance(obj, dict) else obj + + if not key.startswith(base_path): + skipped_count += 1 + continue + + try: + blob = bucket.blob(key) + blob.reload() + current_class = blob.storage_class + + if current_class in ["ARCHIVE", "COLDLINE", "NEARLINE"]: + self.loggit.debug( + "Thawing blob %d/%d: %s from %s to STANDARD", + idx, + len(object_keys), + key, + current_class, + ) + blob.update_storage_class("STANDARD") + restored_count += 1 + else: + self.loggit.debug( + "Skipping blob %d/%d: %s (storage class: %s)", + idx, + len(object_keys), + key, + current_class, + ) + skipped_count += 1 + + except GoogleAPIError as e: + error_count += 1 + self.loggit.error( + "Error thawing blob %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys), + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Move objects to Archive storage class. + + Maps S3 storage classes to GCS: + - GLACIER -> ARCHIVE + - DEEP_ARCHIVE -> ARCHIVE + - GLACIER_IR -> COLDLINE + + Args: + bucket_name (str): The name of the bucket + path (str): The path (prefix) to the objects to refreeze + storage_class (str): The S3-style storage class to move objects to + + Returns: + None + """ + self.loggit.info( + "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class, + ) + + # Map S3 storage class to GCS storage class + class_map = { + "GLACIER": "ARCHIVE", + "DEEP_ARCHIVE": "ARCHIVE", + "GLACIER_IR": "COLDLINE", + } + target_class = class_map.get(storage_class, "ARCHIVE") + + bucket = self.client.bucket(bucket_name) + refrozen_count = 0 + error_count = 0 + + # List blobs with prefix + blobs = bucket.list_blobs(prefix=path) + + for blob in blobs: + try: + current_class = blob.storage_class + self.loggit.debug( + "Refreezing blob: %s (from %s to %s)", + blob.name, + current_class, + target_class, + ) + blob.update_storage_class(target_class) + refrozen_count += 1 + except GoogleAPIError as e: + error_count += 1 + self.loggit.error( + "Error refreezing blob %s: %s (type: %s)", + blob.name, + str(e), + type(e).__name__, + exc_info=True, + ) + + self.loggit.info( + "Refreeze operation completed - refrozen: %d, errors: %d", + refrozen_count, + error_count, + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: + """ + List objects in a bucket with a given prefix. + + Args: + bucket_name (str): The name of the bucket to list objects from. + prefix (str): The prefix to use when listing objects. + + Returns: + list[dict]: A list of object metadata dictionaries with S3-compatible keys. + """ + self.loggit.info( + f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" + ) + + bucket = self.client.bucket(bucket_name) + objects = [] + + try: + blobs = bucket.list_blobs(prefix=prefix) + for blob in blobs: + # Map GCS properties to S3-like structure for compatibility + objects.append( + { + "Key": blob.name, + "Size": blob.size, + "LastModified": blob.updated, + "StorageClass": self._map_gcs_class_to_s3(blob.storage_class), + # GCS-specific + "GcsStorageClass": blob.storage_class, + } + ) + return objects + except GoogleAPIError as e: + self.loggit.error("Error listing objects: %s", e) + raise ActionError(f"Error listing objects: {e}") from e + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete a GCS bucket. + + Args: + bucket_name (str): The name of the bucket to delete. + force (bool): If True, empty the bucket before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting bucket: {bucket_name}") + try: + bucket = self.client.bucket(bucket_name) + + if force: + self.loggit.info(f"Emptying bucket {bucket_name} before deletion") + blobs = bucket.list_blobs() + for blob in blobs: + blob.delete() + self.loggit.debug(f"Deleted blob {blob.name}") + + bucket.delete() + self.loggit.info(f"Bucket {bucket_name} deleted successfully") + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Upload an object to a bucket. + + Args: + bucket_name (str): The name of the bucket to put the object in. + key (str): The key of the object to put. + body (str): The body of the object to put. + + Returns: + None + """ + self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") + try: + bucket = self.client.bucket(bucket_name) + blob = bucket.blob(key) + blob.upload_from_string(body) + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all buckets. + + Args: + prefix (str): Optional prefix to filter bucket names. + + Returns: + list[str]: A list of bucket names. + """ + self.loggit.info("Listing buckets") + try: + buckets = self.client.list_buckets() + bucket_names = [b.name for b in buckets] + if prefix: + bucket_names = [ + name for name in bucket_names if name.startswith(prefix) + ] + return bucket_names + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for an object without downloading it. + + Args: + bucket_name (str): The name of the bucket. + key (str): The object key. + + Returns: + dict: Object metadata including storage class. + """ + self.loggit.debug(f"Getting metadata for gs://{bucket_name}/{key}") + try: + bucket = self.client.bucket(bucket_name) + blob = bucket.blob(key) + blob.reload() + + # Map to S3-like response structure + response = { + "ContentLength": blob.size, + "ContentType": blob.content_type, + "LastModified": blob.updated, + "ETag": blob.etag, + "StorageClass": self._map_gcs_class_to_s3(blob.storage_class), + # GCS doesn't have restore concept like S3 Glacier + # Archive objects are immediately accessible + "Restore": None, + # GCS-specific + "GcsStorageClass": blob.storage_class, + } + return response + except NotFound as e: + self.loggit.error(f"Object not found: {key}") + raise ActionError(f"Object not found: {key}") from e + except GoogleAPIError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") from e + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy an object with storage class change. + + Args: + Bucket (str): The name of the destination bucket. + Key (str): The key for the copied object. + CopySource (dict[str, str]): The source bucket and key. + StorageClass (str): The S3-style storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying object {Key} to bucket {Bucket}") + try: + source_bucket_name = CopySource["Bucket"] + source_key = CopySource["Key"] + + # Map S3 storage class to GCS + class_map = { + "GLACIER": "ARCHIVE", + "DEEP_ARCHIVE": "ARCHIVE", + "STANDARD": "STANDARD", + "STANDARD_IA": "NEARLINE", + } + target_class = class_map.get(StorageClass, "ARCHIVE") + + source_bucket = self.client.bucket(source_bucket_name) + source_blob = source_bucket.blob(source_key) + + dest_bucket = self.client.bucket(Bucket) + dest_blob = dest_bucket.blob(Key) + + # Copy the blob + token = None + while True: + token, _, _ = dest_blob.rewrite(source_blob, token=token) + if token is None: + break + + # Update storage class if different from default + if target_class != "STANDARD": + dest_blob.update_storage_class(target_class) + + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def _map_gcs_class_to_s3(self, gcs_class: str) -> str: + """ + Map GCS storage class to S3 storage class for compatibility. + + Args: + gcs_class (str): The GCS storage class. + + Returns: + str: The equivalent S3 storage class. + """ + class_map = { + "STANDARD": "STANDARD", + "NEARLINE": "STANDARD_IA", + "COLDLINE": "ONEZONE_IA", + "ARCHIVE": "GLACIER", + } + return class_map.get(gcs_class, "STANDARD") diff --git a/packages/deepfreeze-core/deepfreeze_core/s3client.py b/packages/deepfreeze-core/deepfreeze_core/s3client.py index 24ed53c..45e4cef 100644 --- a/packages/deepfreeze-core/deepfreeze_core/s3client.py +++ b/packages/deepfreeze-core/deepfreeze_core/s3client.py @@ -205,8 +205,9 @@ def s3_client_factory(provider: str) -> S3Client: return AzureBlobClient() elif provider == "gcp": - # Placeholder for GCP S3Client implementation - raise NotImplementedError("GCP S3Client is not implemented yet") + from deepfreeze_core.gcp_client import GcpStorageClient + + return GcpStorageClient() else: raise ValueError(f"Unsupported provider: {provider}") diff --git a/packages/deepfreeze-core/pyproject.toml b/packages/deepfreeze-core/pyproject.toml index aac5fad..53d6eb8 100644 --- a/packages/deepfreeze-core/pyproject.toml +++ b/packages/deepfreeze-core/pyproject.toml @@ -45,6 +45,9 @@ dependencies = [ azure = [ "azure-storage-blob>=12.0.0", ] +gcp = [ + "google-cloud-storage>=2.0.0", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", @@ -53,6 +56,7 @@ dev = [ "ruff>=0.1.0", "moto>=4.0.0", "azure-storage-blob>=12.0.0", + "google-cloud-storage>=2.0.0", ] [project.urls] From adcef01a8ab948dbe533177d2ca69127fc397270 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Fri, 16 Jan 2026 11:07:40 -0500 Subject: [PATCH 04/30] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20config=20file?= =?UTF-8?q?=20support=20for=20storage=20provider=20credentials?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - All cloud clients now accept credentials as constructor parameters - Credentials can be specified in config.yaml under 'storage' section - Environment variables remain as fallback if config not provided - Add load_storage_config() and get_storage_credentials() helpers - Update s3_client_factory() to pass **kwargs to provider clients --- .../deepfreeze_core/__init__.py | 4 + .../deepfreeze_core/aws_client.py | 49 +++++++++- .../deepfreeze_core/azure_client.py | 69 ++++++++----- .../deepfreeze_core/esclient.py | 96 +++++++++++++++++++ .../deepfreeze_core/gcp_client.py | 60 +++++++++--- .../deepfreeze_core/s3client.py | 25 ++++- 6 files changed, 261 insertions(+), 42 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/__init__.py b/packages/deepfreeze-core/deepfreeze_core/__init__.py index 18d18f4..bdb18e4 100644 --- a/packages/deepfreeze-core/deepfreeze_core/__init__.py +++ b/packages/deepfreeze-core/deepfreeze_core/__init__.py @@ -33,7 +33,9 @@ ESClientWrapper, create_es_client, create_es_client_from_config, + get_storage_credentials, load_config_from_yaml, + load_storage_config, validate_connection, ) @@ -132,7 +134,9 @@ "ESClientWrapper", "create_es_client", "create_es_client_from_config", + "get_storage_credentials", "load_config_from_yaml", + "load_storage_config", "validate_connection", # Actions "Cleanup", diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py index 9f3f6a1..4d4c961 100644 --- a/packages/deepfreeze-core/deepfreeze_core/aws_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -16,12 +16,57 @@ class AwsS3Client(S3Client): """ An S3 client object for use with AWS. + + Credentials can be provided via constructor arguments or environment variables. + Constructor arguments take precedence over environment variables. + + Args: + region: AWS region (e.g., 'us-east-1') + profile: AWS profile name from ~/.aws/credentials + access_key_id: AWS access key ID + secret_access_key: AWS secret access key + + Environment variables (fallback): + AWS_DEFAULT_REGION: Region + AWS_PROFILE: Profile name + AWS_ACCESS_KEY_ID: Access key ID + AWS_SECRET_ACCESS_KEY: Secret access key """ - def __init__(self) -> None: + def __init__( + self, + region: str = None, + profile: str = None, + access_key_id: str = None, + secret_access_key: str = None, + ) -> None: self.loggit = logging.getLogger("deepfreeze.s3client") try: - self.client = boto3.client("s3") + # Build session/client kwargs based on provided credentials + session_kwargs = {} + client_kwargs = {} + + if profile: + session_kwargs["profile_name"] = profile + self.loggit.debug("Using AWS profile: %s (source: config)", profile) + + if region: + session_kwargs["region_name"] = region + self.loggit.debug("Using AWS region: %s (source: config)", region) + + if access_key_id and secret_access_key: + session_kwargs["aws_access_key_id"] = access_key_id + session_kwargs["aws_secret_access_key"] = secret_access_key + self.loggit.debug("Using explicit AWS credentials (source: config)") + + # Create session and client + if session_kwargs: + session = boto3.Session(**session_kwargs) + self.client = session.client("s3", **client_kwargs) + else: + self.client = boto3.client("s3", **client_kwargs) + self.loggit.debug("Using default AWS credentials (environment/instance)") + # Validate credentials by attempting a simple operation self.loggit.debug("Validating AWS credentials") self.client.list_buckets() diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py index c396f11..6ebb45f 100644 --- a/packages/deepfreeze-core/deepfreeze_core/azure_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -26,36 +26,57 @@ class AzureBlobClient(S3Client): """ Azure Blob Storage client implementing the S3Client interface. + + Credentials can be provided via constructor arguments or environment variables. + Constructor arguments take precedence over environment variables. + + Args: + connection_string: Azure Storage connection string + account_name: Azure Storage account name (used with account_key) + account_key: Azure Storage account key (used with account_name) + + Environment variables (fallback): + AZURE_STORAGE_CONNECTION_STRING: Connection string + AZURE_STORAGE_ACCOUNT: Account name + AZURE_STORAGE_KEY: Account key """ - def __init__(self) -> None: + def __init__( + self, + connection_string: str = None, + account_name: str = None, + account_key: str = None, + ) -> None: self.loggit = logging.getLogger("deepfreeze.azure_client") try: - # Azure SDK uses connection string from environment variable - # AZURE_STORAGE_CONNECTION_STRING or account name + key - connection_string = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") - if connection_string: - self.service_client = BlobServiceClient.from_connection_string( - connection_string + # Priority: constructor args > environment variables + conn_str = connection_string or os.environ.get( + "AZURE_STORAGE_CONNECTION_STRING" + ) + acct_name = account_name or os.environ.get("AZURE_STORAGE_ACCOUNT") + acct_key = account_key or os.environ.get("AZURE_STORAGE_KEY") + + if conn_str: + self.service_client = BlobServiceClient.from_connection_string(conn_str) + self.loggit.debug( + "Using connection string for auth (source: %s)", + "config" if connection_string else "environment", + ) + elif acct_name and acct_key: + account_url = f"https://{acct_name}.blob.core.windows.net" + self.service_client = BlobServiceClient( + account_url=account_url, credential=acct_key + ) + self.loggit.debug( + "Using account name + key for auth (source: %s)", + "config" if account_name else "environment", ) - self.loggit.debug("Using AZURE_STORAGE_CONNECTION_STRING for auth") else: - # Alternative: account name + key - account_name = os.environ.get("AZURE_STORAGE_ACCOUNT") - account_key = os.environ.get("AZURE_STORAGE_KEY") - if account_name and account_key: - account_url = f"https://{account_name}.blob.core.windows.net" - self.service_client = BlobServiceClient( - account_url=account_url, credential=account_key - ) - self.loggit.debug( - "Using AZURE_STORAGE_ACCOUNT + AZURE_STORAGE_KEY for auth" - ) - else: - raise ActionError( - "Azure credentials not configured. Set AZURE_STORAGE_CONNECTION_STRING " - "or both AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY." - ) + raise ActionError( + "Azure credentials not configured. Provide connection_string or " + "account_name + account_key in config, or set environment variables " + "AZURE_STORAGE_CONNECTION_STRING or AZURE_STORAGE_ACCOUNT + AZURE_STORAGE_KEY." + ) # Validate credentials self.loggit.debug("Validating Azure credentials") diff --git a/packages/deepfreeze-core/deepfreeze_core/esclient.py b/packages/deepfreeze-core/deepfreeze_core/esclient.py index 63ba9b7..bfd3737 100644 --- a/packages/deepfreeze-core/deepfreeze_core/esclient.py +++ b/packages/deepfreeze-core/deepfreeze_core/esclient.py @@ -185,6 +185,102 @@ def load_config_from_yaml(config_path: str) -> dict: raise ActionError(f"Error reading configuration file: {e}") from e +def load_storage_config(config_path: str) -> dict: + """ + Load storage provider configuration from a YAML file. + + The YAML file should include a 'storage' section with provider credentials: + + ```yaml + storage: + # AWS S3 configuration + aws: + region: us-east-1 + profile: my-profile # Optional: use named profile + # Or explicit credentials: + # access_key_id: AKIA... + # secret_access_key: ... + + # Azure Blob Storage configuration + azure: + connection_string: "DefaultEndpointsProtocol=https;AccountName=..." + # Or account name + key: + # account_name: mystorageaccount + # account_key: ... + + # Google Cloud Storage configuration + gcp: + project: my-gcp-project + credentials_file: /path/to/service-account.json + location: US + ``` + + Args: + config_path: Path to the YAML configuration file + + Returns: + dict: Storage configuration with provider-specific credentials. + Returns empty dict if no storage section exists. + + Example return value: + { + 'aws': {'region': 'us-east-1', 'profile': 'my-profile'}, + 'azure': {'connection_string': '...'}, + 'gcp': {'project': 'my-project', 'credentials_file': '/path/to/creds.json'} + } + """ + loggit = logging.getLogger("deepfreeze.esclient") + loggit.debug("Loading storage configuration from: %s", config_path) + + config = load_config_from_yaml(config_path) + storage_config = config.get("storage", {}) + + if storage_config: + loggit.debug( + "Found storage configuration for providers: %s", + list(storage_config.keys()), + ) + else: + loggit.debug("No storage configuration found in config file") + + return storage_config + + +def get_storage_credentials(config_path: str, provider: str) -> dict: + """ + Get storage credentials for a specific provider from the config file. + + This is a convenience function that loads the storage config and returns + only the credentials for the specified provider. + + Args: + config_path: Path to the YAML configuration file + provider: The storage provider ('aws', 'azure', or 'gcp') + + Returns: + dict: Provider-specific credentials ready to pass to s3_client_factory. + Returns empty dict if no credentials found for the provider. + + Example: + >>> creds = get_storage_credentials('/path/to/config.yaml', 'azure') + >>> client = s3_client_factory('azure', **creds) + """ + loggit = logging.getLogger("deepfreeze.esclient") + + storage_config = load_storage_config(config_path) + provider_config = storage_config.get(provider, {}) + + if provider_config: + loggit.debug("Found credentials for provider: %s", provider) + else: + loggit.debug( + "No credentials found for provider %s, will use environment variables", + provider, + ) + + return provider_config + + def create_es_client_from_config(config_path: str) -> Elasticsearch: """ Create an Elasticsearch client from a YAML configuration file. diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py index bbd27b2..1f1c8fb 100644 --- a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -22,21 +22,59 @@ class GcpStorageClient(S3Client): """ Google Cloud Storage client implementing the S3Client interface. + + Credentials can be provided via constructor arguments or environment variables. + Constructor arguments take precedence over environment variables. + + Args: + project: GCP project ID + credentials_file: Path to service account JSON credentials file + location: Default location for bucket creation (default: US) + + Environment variables (fallback): + GOOGLE_CLOUD_PROJECT: Project ID + GOOGLE_APPLICATION_CREDENTIALS: Path to credentials file + GOOGLE_CLOUD_LOCATION: Default location for buckets """ - def __init__(self) -> None: + def __init__( + self, + project: str = None, + credentials_file: str = None, + location: str = None, + ) -> None: self.loggit = logging.getLogger("deepfreeze.gcp_client") + self.default_location = location or os.environ.get("GOOGLE_CLOUD_LOCATION", "US") + try: - # GCP SDK uses Application Default Credentials (ADC) - # Set GOOGLE_APPLICATION_CREDENTIALS env var to service account JSON path - # or use gcloud auth application-default login - project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") - if project_id: + # Priority: constructor args > environment variables + project_id = project or os.environ.get("GOOGLE_CLOUD_PROJECT") + creds_file = credentials_file or os.environ.get( + "GOOGLE_APPLICATION_CREDENTIALS" + ) + + if creds_file: + # Use explicit credentials file + self.client = storage.Client.from_service_account_json( + creds_file, project=project_id + ) + self.loggit.debug( + "Using credentials file: %s (source: %s)", + creds_file, + "config" if credentials_file else "environment", + ) + elif project_id: + # Use ADC with explicit project self.client = storage.Client(project=project_id) - self.loggit.debug("Using GOOGLE_CLOUD_PROJECT: %s", project_id) + self.loggit.debug( + "Using project %s with ADC (source: %s)", + project_id, + "config" if project else "environment", + ) else: + # Use default ADC self.client = storage.Client() - self.loggit.debug("Using default project from credentials") + self.loggit.debug("Using default Application Default Credentials") # Validate credentials by listing buckets (limited to 1) self.loggit.debug("Validating GCP credentials") @@ -79,11 +117,9 @@ def create_bucket(self, bucket_name: str) -> None: raise ActionError(f"Bucket {bucket_name} already exists") try: bucket = self.client.bucket(bucket_name) - # Use default location, can be customized via GOOGLE_CLOUD_LOCATION env var - location = os.environ.get("GOOGLE_CLOUD_LOCATION", "US") - self.client.create_bucket(bucket, location=location) + self.client.create_bucket(bucket, location=self.default_location) self.loggit.info( - f"Successfully created bucket {bucket_name} in location {location}" + f"Successfully created bucket {bucket_name} in location {self.default_location}" ) except Conflict as e: raise ActionError(f"Bucket {bucket_name} already exists") from e diff --git a/packages/deepfreeze-core/deepfreeze_core/s3client.py b/packages/deepfreeze-core/deepfreeze_core/s3client.py index 45e4cef..d22c91b 100644 --- a/packages/deepfreeze-core/deepfreeze_core/s3client.py +++ b/packages/deepfreeze-core/deepfreeze_core/s3client.py @@ -180,7 +180,7 @@ def copy_object( return -def s3_client_factory(provider: str) -> S3Client: +def s3_client_factory(provider: str, **kwargs) -> S3Client: """ s3_client_factory method, returns an S3Client object implemented specific to the value of the provider argument. @@ -188,6 +188,23 @@ def s3_client_factory(provider: str) -> S3Client: Args: provider (str): The provider to use for the S3Client object. Should reference an implemented provider (aws, azure, gcp, etc) + **kwargs: Provider-specific configuration options: + + AWS options: + region (str): AWS region (e.g., 'us-east-1') + profile (str): AWS profile name from ~/.aws/credentials + access_key_id (str): AWS access key ID + secret_access_key (str): AWS secret access key + + Azure options: + connection_string (str): Azure Storage connection string + account_name (str): Azure Storage account name + account_key (str): Azure Storage account key + + GCP options: + project (str): GCP project ID + credentials_file (str): Path to service account JSON file + location (str): Default location for bucket creation Raises: NotImplementedError: raised if the provider is not implemented @@ -199,15 +216,15 @@ def s3_client_factory(provider: str) -> S3Client: if provider == "aws": from deepfreeze_core.aws_client import AwsS3Client - return AwsS3Client() + return AwsS3Client(**kwargs) elif provider == "azure": from deepfreeze_core.azure_client import AzureBlobClient - return AzureBlobClient() + return AzureBlobClient(**kwargs) elif provider == "gcp": from deepfreeze_core.gcp_client import GcpStorageClient - return GcpStorageClient() + return GcpStorageClient(**kwargs) else: raise ValueError(f"Unsupported provider: {provider}") From 29e98b83ed14e8209a887d0352a3a9c4aa9bfbf6 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 09:23:27 -0500 Subject: [PATCH 05/30] =?UTF-8?q?=F0=9F=93=9D=20docs:=20update=20documenta?= =?UTF-8?q?tion=20and=20tests=20for=20multi-provider=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enable azure and gcp providers in CLI --provider option - Update all READMEs with multi-provider documentation - Update test mocks to use new aws_client module location - Add tests for azure and gcp client factory methods - Update constants test to expect all three providers 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- README.md | 37 ++++++-- packages/deepfreeze-cli/README.md | 89 ++++++++++++++++--- .../deepfreeze-cli/deepfreeze/cli/main.py | 10 +-- packages/deepfreeze-core/README.md | 89 +++++++++++++++++-- .../deepfreeze_core/aws_client.py | 4 +- .../deepfreeze_core/gcp_client.py | 12 ++- tests/cli/test_exceptions_constants.py | 2 +- tests/cli/test_s3client.py | 72 ++++++++------- 8 files changed, 241 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index a74566e..3c23df1 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,23 @@ pip install git+https://github.com/elastic/deepfreeze.git#subdirectory=packages/ [View deepfreeze-cli documentation](packages/deepfreeze-cli/README.md) +## Supported Cloud Providers + +Deepfreeze supports multiple cloud storage providers: + +| Provider | Storage Type | Archive Tier | +|----------|--------------|--------------| +| **AWS** | S3 | Glacier, Glacier Deep Archive | +| **Azure** | Blob Storage | Archive tier | +| **GCP** | Cloud Storage | Archive storage class | + ## Features -- **Setup**: Configure ILM policies, index templates, and S3 buckets for deepfreeze +- **Setup**: Configure ILM policies, index templates, and storage buckets for deepfreeze - **Rotate**: Create new snapshot repositories on a schedule (weekly/monthly/yearly) - **Status**: View the current state of all deepfreeze components -- **Thaw**: Restore data from Glacier for analysis -- **Refreeze**: Return thawed data to Glacier storage +- **Thaw**: Restore data from archive storage for analysis +- **Refreeze**: Return thawed data to archive storage - **Cleanup**: Remove expired thaw requests and associated resources - **Repair Metadata**: Fix inconsistencies in the deepfreeze status index @@ -53,11 +63,22 @@ pip install git+https://github.com/elastic/deepfreeze.git#subdirectory=packages/ username: elastic password: changeme - deepfreeze: - provider: aws - bucket_name_prefix: my-deepfreeze - repo_name_prefix: deepfreeze - rotate_by: week + # Storage provider credentials (optional - can also use environment variables) + storage: + # AWS S3 + aws: + region: us-east-1 + # profile: my-profile # Or use access_key_id + secret_access_key + + # Azure Blob Storage + azure: + connection_string: "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=..." + # Or use account_name + account_key + + # Google Cloud Storage + gcp: + project: my-gcp-project + credentials_file: /path/to/service-account.json ``` 3. Initialize deepfreeze: diff --git a/packages/deepfreeze-cli/README.md b/packages/deepfreeze-cli/README.md index df35b38..85dd2ae 100644 --- a/packages/deepfreeze-cli/README.md +++ b/packages/deepfreeze-cli/README.md @@ -1,14 +1,22 @@ # Deepfreeze -Standalone Elasticsearch S3 Glacier archival and lifecycle management tool. +Standalone Elasticsearch cloud storage archival and lifecycle management tool. ## Overview -Deepfreeze provides cost-effective S3 Glacier archival and lifecycle management for Elasticsearch snapshot repositories without requiring full Curator installation. It is a lightweight, focused tool for managing long-term data retention in AWS S3 Glacier storage. +Deepfreeze provides cost-effective cloud storage archival and lifecycle management for Elasticsearch snapshot repositories without requiring full Curator installation. It is a lightweight, focused tool for managing long-term data retention in cloud archive storage. + +## Supported Cloud Providers + +| Provider | Storage Type | Archive Tier | +|----------|--------------|--------------| +| **AWS** | S3 | Glacier, Glacier Deep Archive | +| **Azure** | Blob Storage | Archive tier | +| **GCP** | Cloud Storage | Archive storage class | ## Features -- S3 Glacier archival for Elasticsearch snapshot repositories +- Cloud archive storage for Elasticsearch snapshot repositories - Repository rotation with configurable retention - Thaw frozen repositories for data retrieval - Automatic refreeze after data access @@ -52,14 +60,20 @@ deepfreeze --help - Python 3.8 or higher - Elasticsearch 8.x cluster -- AWS credentials configured (for S3 access) +- Cloud provider credentials (one of): + - **AWS**: AWS credentials via environment, config file, or IAM role + - **Azure**: Connection string or account name + key + - **GCP**: Application Default Credentials or service account JSON - Required Python packages (installed automatically): - elasticsearch8 - - boto3 + - boto3 (for AWS) - click - rich - voluptuous - pyyaml +- Optional packages for additional providers: + - azure-storage-blob (for Azure): `pip install deepfreeze-cli[azure]` + - google-cloud-storage (for GCP): `pip install deepfreeze-cli[gcp]` ## Configuration @@ -88,7 +102,7 @@ vim ~/.deepfreeze/config.yml ### Configuration Format -Create a YAML configuration file to specify Elasticsearch connection settings: +Create a YAML configuration file to specify Elasticsearch connection and storage provider settings: ```yaml elasticsearch: @@ -103,6 +117,29 @@ elasticsearch: # Or use Elastic Cloud # cloud_id: deployment:base64string +# Storage provider credentials (optional - can also use environment variables) +storage: + # AWS S3 configuration + aws: + region: us-east-1 + # profile: my-profile # Use named profile from ~/.aws/credentials + # Or explicit credentials: + # access_key_id: AKIA... + # secret_access_key: ... + + # Azure Blob Storage configuration + azure: + connection_string: "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=..." + # Or use account name + key: + # account_name: mystorageaccount + # account_key: ... + + # Google Cloud Storage configuration + gcp: + project: my-gcp-project + credentials_file: /path/to/service-account.json + location: US + logging: loglevel: INFO # logfile: /var/log/deepfreeze.log @@ -112,13 +149,30 @@ logging: Configuration can also be provided via environment variables: +**Elasticsearch:** - `DEEPFREEZE_ES_HOSTS` - Elasticsearch hosts (comma-separated) - `DEEPFREEZE_ES_USERNAME` - Elasticsearch username - `DEEPFREEZE_ES_PASSWORD` - Elasticsearch password - `DEEPFREEZE_ES_API_KEY` - Elasticsearch API key - `DEEPFREEZE_ES_CLOUD_ID` - Elastic Cloud ID -Environment variables override file configuration. +**AWS S3:** +- `AWS_ACCESS_KEY_ID` - AWS access key +- `AWS_SECRET_ACCESS_KEY` - AWS secret key +- `AWS_DEFAULT_REGION` - AWS region +- `AWS_PROFILE` - AWS profile name + +**Azure Blob Storage:** +- `AZURE_STORAGE_CONNECTION_STRING` - Full connection string +- `AZURE_STORAGE_ACCOUNT` - Storage account name (with AZURE_STORAGE_KEY) +- `AZURE_STORAGE_KEY` - Storage account key + +**Google Cloud Storage:** +- `GOOGLE_APPLICATION_CREDENTIALS` - Path to service account JSON +- `GOOGLE_CLOUD_PROJECT` - GCP project ID +- `GOOGLE_CLOUD_LOCATION` - Default bucket location + +Environment variables are used as fallback when config file credentials are not provided. ## Usage @@ -127,11 +181,26 @@ Environment variables override file configuration. Set up deepfreeze with ILM policy and index template configuration: ```bash +# AWS (default) deepfreeze --config config.yaml setup \ --ilm_policy_name my-ilm-policy \ --index_template_name my-template \ --bucket_name_prefix my-deepfreeze \ --repo_name_prefix my-deepfreeze + +# Azure +deepfreeze --config config.yaml setup \ + --provider azure \ + --ilm_policy_name my-ilm-policy \ + --bucket_name_prefix my-deepfreeze \ + --repo_name_prefix my-deepfreeze + +# GCP +deepfreeze --config config.yaml setup \ + --provider gcp \ + --ilm_policy_name my-ilm-policy \ + --bucket_name_prefix my-deepfreeze \ + --repo_name_prefix my-deepfreeze ``` ### Check Status @@ -216,11 +285,11 @@ deepfreeze --config config.yaml --dry-run repair-metadata | Command | Description | |---------|-------------| -| `setup` | Initialize deepfreeze environment | +| `setup` | Initialize deepfreeze environment (supports `--provider` option) | | `status` | Show current status of repositories and requests | | `rotate` | Create new repository and archive old ones | -| `thaw` | Initiate or check Glacier restore operations | -| `refreeze` | Return thawed repositories to Glacier | +| `thaw` | Initiate or check archive restore operations | +| `refreeze` | Return thawed repositories to archive storage | | `cleanup` | Remove expired repositories and old requests | | `repair-metadata` | Scan and repair metadata discrepancies | diff --git a/packages/deepfreeze-cli/deepfreeze/cli/main.py b/packages/deepfreeze-cli/deepfreeze/cli/main.py index 2cd8133..4df2a4b 100644 --- a/packages/deepfreeze-cli/deepfreeze/cli/main.py +++ b/packages/deepfreeze-cli/deepfreeze/cli/main.py @@ -203,15 +203,9 @@ def cli(ctx, config_path, dry_run): @click.option( "-o", "--provider", - type=click.Choice( - [ - "aws", - # "gcp", - # "azure", - ] - ), + type=click.Choice(["aws", "azure", "gcp"]), default="aws", - help="What provider to use (AWS only for now)", + help="Cloud storage provider to use (aws, azure, or gcp)", ) @click.option( "-t", diff --git a/packages/deepfreeze-core/README.md b/packages/deepfreeze-core/README.md index fa171cc..e58a5bc 100644 --- a/packages/deepfreeze-core/README.md +++ b/packages/deepfreeze-core/README.md @@ -1,6 +1,6 @@ # Deepfreeze Core -Core library for Elasticsearch S3 Glacier archival operations. +Core library for Elasticsearch cloud storage archival operations. ## Overview @@ -8,10 +8,28 @@ This package provides the shared functionality for deepfreeze operations, used b - **deepfreeze-cli**: Standalone CLI tool - **elasticsearch-curator**: Full Curator with deepfreeze support +## Supported Cloud Providers + +| Provider | Storage Type | Archive Tier | Package | +|----------|--------------|--------------|---------| +| **AWS** | S3 | Glacier, Deep Archive | boto3 (included) | +| **Azure** | Blob Storage | Archive tier | azure-storage-blob (optional) | +| **GCP** | Cloud Storage | Archive class | google-cloud-storage (optional) | + ## Installation ```bash +# Base installation (AWS support only) pip install deepfreeze-core + +# With Azure support +pip install deepfreeze-core[azure] + +# With GCP support +pip install deepfreeze-core[gcp] + +# With all providers +pip install deepfreeze-core[azure,gcp] ``` ## Usage @@ -21,11 +39,28 @@ This package is typically used as a dependency by other packages. For direct usa ```python from deepfreeze_core import ( Setup, Status, Rotate, Thaw, Refreeze, Cleanup, RepairMetadata, - s3_client_factory, create_es_client + s3_client_factory, create_es_client, get_storage_credentials ) # Create ES client -client = create_es_client(hosts=["https://localhost:9200"], username="elastic", password="changeme") +client = create_es_client( + hosts=["https://localhost:9200"], + username="elastic", + password="changeme" +) + +# Create storage client (AWS) +s3 = s3_client_factory("aws", region="us-east-1") + +# Create storage client (Azure) +s3 = s3_client_factory("azure", connection_string="...") + +# Create storage client (GCP) +s3 = s3_client_factory("gcp", project="my-project") + +# Or load credentials from config file +creds = get_storage_credentials("/path/to/config.yaml", "azure") +s3 = s3_client_factory("azure", **creds) # Create and run an action status = Status(client=client) @@ -37,17 +72,57 @@ status.do_action() ### Actions - `Setup` - Initialize deepfreeze environment - `Status` - Display status of repositories and thaw requests -- `Rotate` - Create new repository, archive old ones to Glacier -- `Thaw` - Restore data from Glacier storage -- `Refreeze` - Return thawed data to Glacier +- `Rotate` - Create new repository, archive old ones +- `Thaw` - Restore data from archive storage +- `Refreeze` - Return thawed data to archive storage - `Cleanup` - Remove expired repositories and requests - `RepairMetadata` - Fix metadata discrepancies +### Storage Clients +- `s3_client_factory(provider, **kwargs)` - Create storage client for any provider +- `AwsS3Client` - AWS S3 implementation +- `AzureBlobClient` - Azure Blob Storage implementation +- `GcpStorageClient` - Google Cloud Storage implementation + ### Utilities -- `s3_client_factory` - Create S3 client for AWS - `create_es_client` - Create Elasticsearch client +- `get_storage_credentials` - Load storage credentials from config file +- `load_storage_config` - Load all storage configuration from config file - Repository and Settings management functions +## Configuration + +Storage credentials can be provided via: + +1. **Constructor arguments** (highest priority) +2. **Config file** (storage section) +3. **Environment variables** (fallback) + +### Config File Format + +```yaml +storage: + aws: + region: us-east-1 + profile: my-profile # or access_key_id + secret_access_key + + azure: + connection_string: "DefaultEndpointsProtocol=https;..." + # or account_name + account_key + + gcp: + project: my-project + credentials_file: /path/to/service-account.json +``` + +### Environment Variables + +**AWS:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `AWS_PROFILE` + +**Azure:** `AZURE_STORAGE_CONNECTION_STRING` or `AZURE_STORAGE_ACCOUNT` + `AZURE_STORAGE_KEY` + +**GCP:** `GOOGLE_APPLICATION_CREDENTIALS`, `GOOGLE_CLOUD_PROJECT` + ## Development ```bash diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py index 4d4c961..9451e40 100644 --- a/packages/deepfreeze-core/deepfreeze_core/aws_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -65,7 +65,9 @@ def __init__( self.client = session.client("s3", **client_kwargs) else: self.client = boto3.client("s3", **client_kwargs) - self.loggit.debug("Using default AWS credentials (environment/instance)") + self.loggit.debug( + "Using default AWS credentials (environment/instance)" + ) # Validate credentials by attempting a simple operation self.loggit.debug("Validating AWS credentials") diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py index 1f1c8fb..02ef675 100644 --- a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -44,7 +44,9 @@ def __init__( location: str = None, ) -> None: self.loggit = logging.getLogger("deepfreeze.gcp_client") - self.default_location = location or os.environ.get("GOOGLE_CLOUD_LOCATION", "US") + self.default_location = location or os.environ.get( + "GOOGLE_CLOUD_LOCATION", "US" + ) try: # Priority: constructor args > environment variables @@ -83,16 +85,12 @@ def __init__( except GoogleAPIError as e: self.loggit.error("Failed to initialize GCP Storage Client: %s", e) - raise ActionError( - f"Failed to initialize GCP Storage Client: {e}" - ) from e + raise ActionError(f"Failed to initialize GCP Storage Client: {e}") from e except Exception as e: self.loggit.error( "Failed to initialize GCP Storage Client: %s", e, exc_info=True ) - raise ActionError( - f"Failed to initialize GCP Storage Client: {e}" - ) from e + raise ActionError(f"Failed to initialize GCP Storage Client: {e}") from e def test_connection(self) -> bool: """ diff --git a/tests/cli/test_exceptions_constants.py b/tests/cli/test_exceptions_constants.py index 52e1522..56427d8 100644 --- a/tests/cli/test_exceptions_constants.py +++ b/tests/cli/test_exceptions_constants.py @@ -80,7 +80,7 @@ def test_constants_importable(): # Verify values assert STATUS_INDEX == "deepfreeze-status" assert SETTINGS_ID == "1" - assert PROVIDERS == ["aws"] + assert PROVIDERS == ["aws", "azure", "gcp"] # Verify thaw states assert THAW_STATE_ACTIVE == "active" diff --git a/tests/cli/test_s3client.py b/tests/cli/test_s3client.py index eefaea5..3b3be9d 100644 --- a/tests/cli/test_s3client.py +++ b/tests/cli/test_s3client.py @@ -24,7 +24,7 @@ class TestAwsS3ClientInstantiation: """Tests for AwsS3Client instantiation""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_client_instantiation_success(self, mock_boto3): """Test that AwsS3Client can be instantiated with valid credentials""" from deepfreeze import AwsS3Client @@ -41,7 +41,7 @@ def test_client_instantiation_success(self, mock_boto3): mock_boto3.client.assert_called_once_with("s3") mock_client.list_buckets.assert_called_once() - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_client_instantiation_invalid_credentials(self, mock_boto3): """Test that AwsS3Client raises ActionError for invalid credentials""" from botocore.exceptions import ClientError @@ -66,7 +66,7 @@ def test_client_instantiation_invalid_credentials(self, mock_boto3): class TestBucketExists: """Tests for bucket_exists method""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_bucket_exists_true(self, mock_boto3): """Test bucket_exists returns True when bucket exists""" from deepfreeze import AwsS3Client @@ -82,7 +82,7 @@ def test_bucket_exists_true(self, mock_boto3): assert result is True mock_client.head_bucket.assert_called_once_with(Bucket="test-bucket") - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_bucket_exists_false(self, mock_boto3): """Test bucket_exists returns False when bucket does not exist""" from botocore.exceptions import ClientError @@ -103,7 +103,7 @@ def test_bucket_exists_false(self, mock_boto3): class TestListObjects: """Tests for list_objects method""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_list_objects_empty(self, mock_boto3): """Test list_objects returns empty list when no objects""" from deepfreeze import AwsS3Client @@ -122,7 +122,7 @@ def test_list_objects_empty(self, mock_boto3): assert result == [] - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_list_objects_with_results(self, mock_boto3): """Test list_objects returns objects when they exist""" from deepfreeze import AwsS3Client @@ -151,7 +151,7 @@ def test_list_objects_with_results(self, mock_boto3): class TestS3ClientFactory: """Tests for s3_client_factory function""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_factory_returns_aws_client(self, mock_boto3): """Test factory returns AwsS3Client for 'aws' provider""" from deepfreeze import AwsS3Client, s3_client_factory @@ -164,23 +164,31 @@ def test_factory_returns_aws_client(self, mock_boto3): assert isinstance(client, AwsS3Client) - def test_factory_raises_not_implemented_for_gcp(self): - """Test factory raises NotImplementedError for 'gcp' provider""" + @patch("deepfreeze_core.gcp_client.storage") + def test_factory_returns_gcp_client(self, mock_storage): + """Test factory returns GcpStorageClient for 'gcp' provider""" from deepfreeze import s3_client_factory + from deepfreeze_core import GcpStorageClient - with pytest.raises(NotImplementedError) as exc_info: - s3_client_factory("gcp") + mock_gcp_client = MagicMock() + mock_storage.Client.return_value = mock_gcp_client - assert "GCP S3Client is not implemented" in str(exc_info.value) + client = s3_client_factory("gcp") - def test_factory_raises_not_implemented_for_azure(self): - """Test factory raises NotImplementedError for 'azure' provider""" + assert isinstance(client, GcpStorageClient) + + @patch("deepfreeze_core.azure_client.BlobServiceClient") + def test_factory_returns_azure_client(self, mock_blob_service): + """Test factory returns AzureBlobClient for 'azure' provider""" from deepfreeze import s3_client_factory + from deepfreeze_core import AzureBlobClient + + mock_azure_client = MagicMock() + mock_blob_service.from_connection_string.return_value = mock_azure_client - with pytest.raises(NotImplementedError) as exc_info: - s3_client_factory("azure") + client = s3_client_factory("azure", connection_string="test-connection-string") - assert "Azure S3Client is not implemented" in str(exc_info.value) + assert isinstance(client, AzureBlobClient) def test_factory_raises_value_error_for_invalid_provider(self): """Test factory raises ValueError for invalid provider""" @@ -195,7 +203,7 @@ def test_factory_raises_value_error_for_invalid_provider(self): class TestS3ClientOperations: """Tests for various S3Client operations""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_test_connection_success(self, mock_boto3): """Test test_connection returns True when connection succeeds""" from deepfreeze import AwsS3Client @@ -209,7 +217,7 @@ def test_test_connection_success(self, mock_boto3): assert result is True - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_test_connection_failure(self, mock_boto3): """Test test_connection returns False when connection fails""" from botocore.exceptions import ClientError @@ -237,7 +245,7 @@ def test_test_connection_failure(self, mock_boto3): class TestBucketOperations: """Tests for bucket create and delete operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_create_bucket_success(self, mock_boto3): """Test create_bucket creates bucket successfully""" from deepfreeze import AwsS3Client @@ -262,7 +270,7 @@ def test_create_bucket_success(self, mock_boto3): mock_client.create_bucket.assert_called_once_with(Bucket="new-bucket") - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_create_bucket_already_exists(self, mock_boto3): """Test create_bucket raises error when bucket exists""" from deepfreeze import ActionError, AwsS3Client @@ -279,7 +287,7 @@ def test_create_bucket_already_exists(self, mock_boto3): assert "already exists" in str(exc_info.value) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_delete_bucket_success(self, mock_boto3): """Test delete_bucket deletes bucket successfully""" from deepfreeze import AwsS3Client @@ -293,7 +301,7 @@ def test_delete_bucket_success(self, mock_boto3): mock_client.delete_bucket.assert_called_once_with(Bucket="delete-me-bucket") - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_delete_bucket_force(self, mock_boto3): """Test delete_bucket with force=True empties bucket first""" from deepfreeze import AwsS3Client @@ -318,7 +326,7 @@ def test_delete_bucket_force(self, mock_boto3): class TestObjectOperations: """Tests for object put, head, and copy operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_put_object_success(self, mock_boto3): """Test put_object successfully puts object""" from deepfreeze import AwsS3Client @@ -334,7 +342,7 @@ def test_put_object_success(self, mock_boto3): Bucket="test-bucket", Key="test-key", Body="test-body" ) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_head_object_success(self, mock_boto3): """Test head_object returns metadata""" from deepfreeze import AwsS3Client @@ -357,7 +365,7 @@ def test_head_object_success(self, mock_boto3): Bucket="test-bucket", Key="test-key" ) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_copy_object_success(self, mock_boto3): """Test copy_object copies with storage class""" from deepfreeze import AwsS3Client @@ -381,7 +389,7 @@ def test_copy_object_success(self, mock_boto3): StorageClass="GLACIER", ) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_list_buckets_with_prefix(self, mock_boto3): """Test list_buckets filters by prefix""" from deepfreeze import AwsS3Client @@ -408,7 +416,7 @@ def test_list_buckets_with_prefix(self, mock_boto3): class TestThawRefreezeOperations: """Tests for thaw and refreeze operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_thaw_initiates_restore_for_glacier_objects(self, mock_boto3): """Test thaw initiates restore for Glacier storage class objects""" from deepfreeze import AwsS3Client @@ -436,7 +444,7 @@ def test_thaw_initiates_restore_for_glacier_objects(self, mock_boto3): # Should have called restore_object twice (only for GLACIER and DEEP_ARCHIVE) assert mock_client.restore_object.call_count == 2 - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_thaw_skips_objects_outside_base_path(self, mock_boto3): """Test thaw skips objects that don't start with base_path""" from deepfreeze import AwsS3Client @@ -463,7 +471,7 @@ def test_thaw_skips_objects_outside_base_path(self, mock_boto3): # Should have called restore_object only once assert mock_client.restore_object.call_count == 1 - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_refreeze_copies_objects_to_glacier(self, mock_boto3): """Test refreeze copies objects to GLACIER storage class""" from deepfreeze import AwsS3Client @@ -493,7 +501,7 @@ def test_refreeze_copies_objects_to_glacier(self, mock_boto3): class TestS3ClientErrorHandling: """Tests for error handling in S3 operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_head_object_raises_action_error(self, mock_boto3): """Test head_object raises ActionError on failure""" from botocore.exceptions import ClientError @@ -513,7 +521,7 @@ def test_head_object_raises_action_error(self, mock_boto3): assert "Error getting metadata" in str(exc_info.value) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_bucket_exists_raises_on_unexpected_error(self, mock_boto3): """Test bucket_exists raises ActionError on unexpected errors""" from botocore.exceptions import ClientError From e3891104e08919f36186a2a49a11fe36b3d066cf Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 10:39:10 -0500 Subject: [PATCH 06/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20use=20correct=20Azu?= =?UTF-8?q?re=20SDK=20parameter=20name=20for=20list=5Fcontainers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change max_results to results_per_page for azure-storage-blob 12.x compatibility. --- packages/deepfreeze-core/deepfreeze_core/azure_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py index 6ebb45f..eba8429 100644 --- a/packages/deepfreeze-core/deepfreeze_core/azure_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -80,7 +80,8 @@ def __init__( # Validate credentials self.loggit.debug("Validating Azure credentials") - list(self.service_client.list_containers(max_results=1)) + # Use results_per_page (azure-storage-blob 12.x parameter name) + list(self.service_client.list_containers(results_per_page=1)) self.loggit.info("Azure Blob Storage Client initialized successfully") except AzureError as e: @@ -105,7 +106,7 @@ def test_connection(self) -> bool: """ try: self.loggit.debug("Testing Azure connection") - list(self.service_client.list_containers(max_results=1)) + list(self.service_client.list_containers(results_per_page=1)) return True except AzureError as e: self.loggit.error("Azure connection test failed: %s", e) From dc3658d329fd18c97f79cd5d3a6d9ffb65bbb4c6 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 10:43:12 -0500 Subject: [PATCH 07/30] =?UTF-8?q?=E2=9C=A8=20feat:=20prompt=20to=20convert?= =?UTF-8?q?=20underscores=20in=20Azure=20container=20names?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Azure Blob Storage containers don't allow underscores in names. When using --provider azure, the CLI now detects underscores in bucket_name_prefix, repo_name_prefix, and base_path_prefix, and prompts the user to confirm converting them to hyphens before proceeding. --- .../deepfreeze-cli/deepfreeze/cli/main.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/deepfreeze-cli/deepfreeze/cli/main.py b/packages/deepfreeze-cli/deepfreeze/cli/main.py index 4df2a4b..3431a24 100644 --- a/packages/deepfreeze-cli/deepfreeze/cli/main.py +++ b/packages/deepfreeze-cli/deepfreeze/cli/main.py @@ -300,6 +300,42 @@ def setup( client = get_client_from_context(ctx) + # Azure container names don't allow underscores - offer to convert them + if provider == "azure": + names_to_check = { + "bucket_name_prefix": bucket_name_prefix, + "repo_name_prefix": repo_name_prefix, + "base_path_prefix": base_path_prefix, + } + names_with_underscores = { + name: value + for name, value in names_to_check.items() + if value and "_" in value + } + if names_with_underscores: + converted = { + name: value.replace("_", "-") + for name, value in names_with_underscores.items() + } + click.echo( + "Azure container names cannot contain underscores. " + "The following names would be converted:" + ) + for name, value in names_with_underscores.items(): + click.echo(f" {name}: {value} -> {converted[name]}") + + if not click.confirm("Do you want to proceed with these converted names?"): + click.echo("Aborted. Please provide names without underscores.") + ctx.exit(1) + + # Apply conversions + if "bucket_name_prefix" in converted: + bucket_name_prefix = converted["bucket_name_prefix"] + if "repo_name_prefix" in converted: + repo_name_prefix = converted["repo_name_prefix"] + if "base_path_prefix" in converted: + base_path_prefix = converted["base_path_prefix"] + action = Setup( client=client, year=year, From 635800392f982f39b0b328e14a842a6209a7132e Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:00:21 -0500 Subject: [PATCH 08/30] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20provider-specif?= =?UTF-8?q?ic=20error=20messages=20for=20repository=20setup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update repository creation error to show provider-specific solutions - Include plugin installation commands for AWS/Azure/GCP - Add links to Elastic documentation for each provider's repository plugin - Update preconditions check to verify correct plugin based on provider - Update storage bucket creation errors to be provider-aware --- .../deepfreeze_core/actions/setup.py | 153 ++++++++++++++---- 1 file changed, 118 insertions(+), 35 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py index a8d2e25..ca663a5 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py @@ -249,9 +249,22 @@ def _check_preconditions(self) -> None: template_type, ) - # Fifth, check for S3 repository plugin (only for ES 7.x and below) - # NOTE: Elasticsearch 8.x+ has built-in S3 repository support, no plugin needed - self.loggit.debug("Checking S3 repository support") + # Fifth, check for repository plugin based on provider + # NOTE: Elasticsearch 8.x+ has built-in repository support for all providers + provider = self.settings.provider + plugin_map = { + "aws": ("repository-s3", "S3"), + "azure": ("repository-azure", "Azure"), + "gcp": ("repository-gcs", "GCS"), + } + plugin_name, plugin_display = plugin_map.get(provider, ("repository-s3", "S3")) + doc_urls = { + "aws": "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html", + "azure": "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure.html", + "gcp": "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html", + } + + self.loggit.debug("Checking %s repository support", plugin_display) try: # Get Elasticsearch version cluster_info = self.client.info() @@ -259,47 +272,55 @@ def _check_preconditions(self) -> None: major_version = int(es_version.split(".")[0]) if major_version < 8: - # ES 7.x and below require the repository-s3 plugin + # ES 7.x and below require repository plugins self.loggit.debug( - "Elasticsearch %s detected - checking for S3 repository plugin", + "Elasticsearch %s detected - checking for %s repository plugin", es_version, + plugin_display, ) # Get cluster plugins nodes_info = self.client.nodes.info(node_id="_all", metric="plugins") - # Check if any node has the S3 plugin - has_s3_plugin = False + # Check if any node has the required plugin + has_plugin = False for node_id, node_data in nodes_info.get("nodes", {}).items(): plugins = node_data.get("plugins", []) for plugin in plugins: - if plugin.get("name") == "repository-s3": - has_s3_plugin = True - self.loggit.debug("Found S3 plugin on node %s", node_id) + if plugin.get("name") == plugin_name: + has_plugin = True + self.loggit.debug( + "Found %s plugin on node %s", plugin_name, node_id + ) break - if has_s3_plugin: + if has_plugin: break - if not has_s3_plugin: + if not has_plugin: errors.append( { - "issue": "Elasticsearch S3 repository plugin is not installed", - "solution": "Install the S3 repository plugin on all Elasticsearch nodes:\n" - " [yellow]bin/elasticsearch-plugin install repository-s3[/yellow]\n" + "issue": f"Elasticsearch {plugin_display} repository plugin is not installed", + "solution": f"Install the {plugin_display} repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install {plugin_name}[/yellow]\n" " Then restart all Elasticsearch nodes.\n" - " See: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html", + f" See: {doc_urls[provider]}", } ) else: - self.loggit.debug("S3 repository plugin is installed") + self.loggit.debug( + "%s repository plugin is installed", plugin_display + ) else: - # ES 8.x+ has built-in S3 support + # ES 8.x+ has built-in repository support self.loggit.debug( - "Elasticsearch %s detected - S3 repository support is built-in", + "Elasticsearch %s detected - %s repository support is built-in", es_version, + plugin_display, ) except Exception as e: - self.loggit.warning("Could not verify S3 repository support: %s", e) + self.loggit.warning( + "Could not verify %s repository support: %s", plugin_display, e + ) # Don't add to errors - this is a soft check that may fail due to permissions # If any errors were found, display them all and raise exception @@ -433,19 +454,45 @@ def do_action(self) -> None: ) except Exception as e: if self.porcelain: - print(f"ERROR\ts3_bucket\t{self.new_bucket_name}\t{str(e)}") + print(f"ERROR\tstorage\t{self.new_bucket_name}\t{str(e)}") else: + # Provider-specific error messages + provider = self.settings.provider + if provider == "azure": + storage_type = "Azure container" + solutions = ( + "[bold]Possible Solutions:[/bold]\n" + " - Check Azure credentials (connection string or account name + key)\n" + " - Verify the storage account exists and is accessible\n" + " - Check if container name is valid (lowercase, no underscores)\n" + " - Verify Azure RBAC permissions allow container creation" + ) + elif provider == "gcp": + storage_type = "GCS bucket" + solutions = ( + "[bold]Possible Solutions:[/bold]\n" + " - Check GCP credentials (service account JSON)\n" + " - Verify the GCP project is correctly configured\n" + " - Check if bucket name is globally unique\n" + " - Verify IAM permissions allow storage.buckets.create" + ) + else: # aws + storage_type = "S3 bucket" + solutions = ( + "[bold]Possible Solutions:[/bold]\n" + " - Check AWS credentials and permissions\n" + " - Verify IAM policy allows s3:CreateBucket\n" + " - Check if bucket name is globally unique\n" + " - Verify AWS region settings\n" + " - Check AWS account limits for S3 buckets" + ) + self.console.print( Panel( - f"[bold]Failed to create S3 bucket [cyan]{self.new_bucket_name}[/cyan][/bold]\n\n" + f"[bold]Failed to create {storage_type} [cyan]{self.new_bucket_name}[/cyan][/bold]\n\n" f"Error: {escape(str(e))}\n\n" - f"[bold]Possible Solutions:[/bold]\n" - f" - Check AWS credentials and permissions\n" - f" - Verify IAM policy allows s3:CreateBucket\n" - f" - Check if bucket name is globally unique\n" - f" - Verify AWS region settings\n" - f" - Check AWS account limits for S3 buckets", - title="[bold red]S3 Bucket Creation Error[/bold red]", + f"{solutions}", + title=f"[bold red]{storage_type.title()} Creation Error[/bold red]", border_style="red", expand=False, ) @@ -479,16 +526,52 @@ def do_action(self) -> None: if self.porcelain: print(f"ERROR\trepository\t{self.new_repo_name}\t{str(e)}") else: + # Provider-specific error messages and documentation + provider = self.settings.provider + if provider == "azure": + solutions = ( + f"[bold]Possible Solutions:[/bold]\n" + f" 1. Install the Azure repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install repository-azure[/yellow]\n\n" + f" 2. Configure Azure credentials in Elasticsearch keystore:\n" + f" [yellow]bin/elasticsearch-keystore add azure.client.default.account[/yellow]\n" + f" [yellow]bin/elasticsearch-keystore add azure.client.default.key[/yellow]\n\n" + f" 3. Restart Elasticsearch after configuring the keystore\n\n" + f" 4. Verify Azure container [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" + f"[bold]Documentation:[/bold]\n" + f" https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure.html" + ) + elif provider == "gcp": + solutions = ( + f"[bold]Possible Solutions:[/bold]\n" + f" 1. Install the GCS repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install repository-gcs[/yellow]\n\n" + f" 2. Configure GCP credentials in Elasticsearch keystore:\n" + f" [yellow]bin/elasticsearch-keystore add-file gcs.client.default.credentials_file /path/to/service-account.json[/yellow]\n\n" + f" 3. Restart Elasticsearch after configuring the keystore\n\n" + f" 4. Verify GCS bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" + f"[bold]Documentation:[/bold]\n" + f" https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html" + ) + else: # aws + solutions = ( + f"[bold]Possible Solutions:[/bold]\n" + f" 1. Install the S3 repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install repository-s3[/yellow]\n\n" + f" 2. Configure AWS credentials in Elasticsearch keystore:\n" + f" [yellow]bin/elasticsearch-keystore add s3.client.default.access_key[/yellow]\n" + f" [yellow]bin/elasticsearch-keystore add s3.client.default.secret_key[/yellow]\n\n" + f" 3. Restart Elasticsearch after configuring the keystore\n\n" + f" 4. Verify S3 bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" + f"[bold]Documentation:[/bold]\n" + f" https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" + ) + self.console.print( Panel( f"[bold]Failed to create repository [cyan]{self.new_repo_name}[/cyan][/bold]\n\n" f"Error: {escape(str(e))}\n\n" - f"[bold]Possible Solutions:[/bold]\n" - f" - Verify Elasticsearch has S3 plugin installed\n" - f" - Check AWS credentials are configured in Elasticsearch keystore\n" - f" - Verify S3 bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n" - f" - Check repository settings (ACL, storage class, etc.)\n" - f" - Review Elasticsearch logs for detailed error messages", + f"{solutions}", title="[bold red]Repository Creation Error[/bold red]", border_style="red", expand=False, From f2ac482f88df7a587bbb49ac57824f0154873ccc Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:02:48 -0500 Subject: [PATCH 09/30] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20move=20p?= =?UTF-8?q?rovider-specific=20error=20info=20to=20client=20classes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ES_PLUGIN_NAME, ES_PLUGIN_DOC_URL, STORAGE_TYPE, etc. class attributes to AwsS3Client, AzureBlobClient, and GcpStorageClient - Update setup action to read error help text from client class - Add class attributes to MockS3Client for test compatibility This keeps provider-specific information in the provider modules rather than hardcoding it in the setup action. --- .../deepfreeze_core/actions/setup.py | 109 ++++-------------- .../deepfreeze_core/aws_client.py | 21 ++++ .../deepfreeze_core/azure_client.py | 20 ++++ .../deepfreeze_core/gcp_client.py | 19 +++ tests/cli/test_integration.py | 8 ++ 5 files changed, 93 insertions(+), 84 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py index ca663a5..e4a7481 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py @@ -251,18 +251,10 @@ def _check_preconditions(self) -> None: # Fifth, check for repository plugin based on provider # NOTE: Elasticsearch 8.x+ has built-in repository support for all providers - provider = self.settings.provider - plugin_map = { - "aws": ("repository-s3", "S3"), - "azure": ("repository-azure", "Azure"), - "gcp": ("repository-gcs", "GCS"), - } - plugin_name, plugin_display = plugin_map.get(provider, ("repository-s3", "S3")) - doc_urls = { - "aws": "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html", - "azure": "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure.html", - "gcp": "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html", - } + # Get plugin info from the storage client class + plugin_name = self.s3.ES_PLUGIN_NAME + plugin_display = self.s3.ES_PLUGIN_DISPLAY_NAME + doc_url = self.s3.ES_PLUGIN_DOC_URL self.loggit.debug("Checking %s repository support", plugin_display) try: @@ -303,7 +295,7 @@ def _check_preconditions(self) -> None: "solution": f"Install the {plugin_display} repository plugin on all Elasticsearch nodes:\n" f" [yellow]bin/elasticsearch-plugin install {plugin_name}[/yellow]\n" " Then restart all Elasticsearch nodes.\n" - f" See: {doc_urls[provider]}", + f" See: {doc_url}", } ) else: @@ -456,42 +448,15 @@ def do_action(self) -> None: if self.porcelain: print(f"ERROR\tstorage\t{self.new_bucket_name}\t{str(e)}") else: - # Provider-specific error messages - provider = self.settings.provider - if provider == "azure": - storage_type = "Azure container" - solutions = ( - "[bold]Possible Solutions:[/bold]\n" - " - Check Azure credentials (connection string or account name + key)\n" - " - Verify the storage account exists and is accessible\n" - " - Check if container name is valid (lowercase, no underscores)\n" - " - Verify Azure RBAC permissions allow container creation" - ) - elif provider == "gcp": - storage_type = "GCS bucket" - solutions = ( - "[bold]Possible Solutions:[/bold]\n" - " - Check GCP credentials (service account JSON)\n" - " - Verify the GCP project is correctly configured\n" - " - Check if bucket name is globally unique\n" - " - Verify IAM permissions allow storage.buckets.create" - ) - else: # aws - storage_type = "S3 bucket" - solutions = ( - "[bold]Possible Solutions:[/bold]\n" - " - Check AWS credentials and permissions\n" - " - Verify IAM policy allows s3:CreateBucket\n" - " - Check if bucket name is globally unique\n" - " - Verify AWS region settings\n" - " - Check AWS account limits for S3 buckets" - ) + # Get provider-specific error info from the storage client + storage_type = self.s3.STORAGE_TYPE + solutions = self.s3.STORAGE_CREATION_HELP self.console.print( Panel( f"[bold]Failed to create {storage_type} [cyan]{self.new_bucket_name}[/cyan][/bold]\n\n" f"Error: {escape(str(e))}\n\n" - f"{solutions}", + f"[bold]{solutions}[/bold]", title=f"[bold red]{storage_type.title()} Creation Error[/bold red]", border_style="red", expand=False, @@ -526,46 +491,22 @@ def do_action(self) -> None: if self.porcelain: print(f"ERROR\trepository\t{self.new_repo_name}\t{str(e)}") else: - # Provider-specific error messages and documentation - provider = self.settings.provider - if provider == "azure": - solutions = ( - f"[bold]Possible Solutions:[/bold]\n" - f" 1. Install the Azure repository plugin on all Elasticsearch nodes:\n" - f" [yellow]bin/elasticsearch-plugin install repository-azure[/yellow]\n\n" - f" 2. Configure Azure credentials in Elasticsearch keystore:\n" - f" [yellow]bin/elasticsearch-keystore add azure.client.default.account[/yellow]\n" - f" [yellow]bin/elasticsearch-keystore add azure.client.default.key[/yellow]\n\n" - f" 3. Restart Elasticsearch after configuring the keystore\n\n" - f" 4. Verify Azure container [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" - f"[bold]Documentation:[/bold]\n" - f" https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure.html" - ) - elif provider == "gcp": - solutions = ( - f"[bold]Possible Solutions:[/bold]\n" - f" 1. Install the GCS repository plugin on all Elasticsearch nodes:\n" - f" [yellow]bin/elasticsearch-plugin install repository-gcs[/yellow]\n\n" - f" 2. Configure GCP credentials in Elasticsearch keystore:\n" - f" [yellow]bin/elasticsearch-keystore add-file gcs.client.default.credentials_file /path/to/service-account.json[/yellow]\n\n" - f" 3. Restart Elasticsearch after configuring the keystore\n\n" - f" 4. Verify GCS bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" - f"[bold]Documentation:[/bold]\n" - f" https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html" - ) - else: # aws - solutions = ( - f"[bold]Possible Solutions:[/bold]\n" - f" 1. Install the S3 repository plugin on all Elasticsearch nodes:\n" - f" [yellow]bin/elasticsearch-plugin install repository-s3[/yellow]\n\n" - f" 2. Configure AWS credentials in Elasticsearch keystore:\n" - f" [yellow]bin/elasticsearch-keystore add s3.client.default.access_key[/yellow]\n" - f" [yellow]bin/elasticsearch-keystore add s3.client.default.secret_key[/yellow]\n\n" - f" 3. Restart Elasticsearch after configuring the keystore\n\n" - f" 4. Verify S3 bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" - f"[bold]Documentation:[/bold]\n" - f" https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" - ) + # Get provider-specific error info from the storage client + plugin_name = self.s3.ES_PLUGIN_NAME + plugin_display = self.s3.ES_PLUGIN_DISPLAY_NAME + doc_url = self.s3.ES_PLUGIN_DOC_URL + storage_type = self.s3.STORAGE_TYPE + keystore_instructions = self.s3.ES_KEYSTORE_INSTRUCTIONS + + solutions = ( + f"[bold]Possible Solutions:[/bold]\n" + f" 1. Install the {plugin_display} repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install {plugin_name}[/yellow]\n\n" + f" 2. {keystore_instructions}\n\n" + f" 3. Verify {storage_type} [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" + f"[bold]Documentation:[/bold]\n" + f" {doc_url}" + ) self.console.print( Panel( diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py index 9451e40..546c0ad 100644 --- a/packages/deepfreeze-core/deepfreeze_core/aws_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -33,6 +33,27 @@ class AwsS3Client(S3Client): AWS_SECRET_ACCESS_KEY: Secret access key """ + # Elasticsearch repository plugin information + ES_PLUGIN_NAME = "repository-s3" + ES_PLUGIN_DISPLAY_NAME = "S3" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" + STORAGE_TYPE = "S3 bucket" + + # Elasticsearch keystore setup instructions + ES_KEYSTORE_INSTRUCTIONS = """Configure AWS credentials in Elasticsearch keystore: + bin/elasticsearch-keystore add s3.client.default.access_key + bin/elasticsearch-keystore add s3.client.default.secret_key + +Then restart Elasticsearch to apply the keystore changes.""" + + # Storage bucket creation error help + STORAGE_CREATION_HELP = """Possible solutions: + - Check AWS credentials and permissions + - Verify IAM policy allows s3:CreateBucket + - Check if bucket name is globally unique + - Verify AWS region settings + - Check AWS account limits for S3 buckets""" + def __init__( self, region: str = None, diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py index eba8429..6684344 100644 --- a/packages/deepfreeze-core/deepfreeze_core/azure_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -41,6 +41,26 @@ class AzureBlobClient(S3Client): AZURE_STORAGE_KEY: Account key """ + # Elasticsearch repository plugin information + ES_PLUGIN_NAME = "repository-azure" + ES_PLUGIN_DISPLAY_NAME = "Azure" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure.html" + STORAGE_TYPE = "Azure container" + + # Elasticsearch keystore setup instructions + ES_KEYSTORE_INSTRUCTIONS = """Configure Azure credentials in Elasticsearch keystore: + bin/elasticsearch-keystore add azure.client.default.account + bin/elasticsearch-keystore add azure.client.default.key + +Then restart Elasticsearch to apply the keystore changes.""" + + # Storage bucket creation error help + STORAGE_CREATION_HELP = """Possible solutions: + - Check Azure credentials (connection string or account name + key) + - Verify the storage account exists and is accessible + - Check if container name is valid (lowercase, no underscores, 3-63 chars) + - Verify Azure RBAC permissions allow container creation""" + def __init__( self, connection_string: str = None, diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py index 02ef675..e5ae4a3 100644 --- a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -37,6 +37,25 @@ class GcpStorageClient(S3Client): GOOGLE_CLOUD_LOCATION: Default location for buckets """ + # Elasticsearch repository plugin information + ES_PLUGIN_NAME = "repository-gcs" + ES_PLUGIN_DISPLAY_NAME = "GCS" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html" + STORAGE_TYPE = "GCS bucket" + + # Elasticsearch keystore setup instructions + ES_KEYSTORE_INSTRUCTIONS = """Configure GCP credentials in Elasticsearch keystore: + bin/elasticsearch-keystore add-file gcs.client.default.credentials_file /path/to/service-account.json + +Then restart Elasticsearch to apply the keystore changes.""" + + # Storage bucket creation error help + STORAGE_CREATION_HELP = """Possible solutions: + - Check GCP credentials (service account JSON file) + - Verify the GCP project is correctly configured + - Check if bucket name is globally unique + - Verify IAM permissions allow storage.buckets.create""" + def __init__( self, project: str = None, diff --git a/tests/cli/test_integration.py b/tests/cli/test_integration.py index b39ff83..7eeec21 100644 --- a/tests/cli/test_integration.py +++ b/tests/cli/test_integration.py @@ -112,6 +112,14 @@ def _search_documents(self, index=None, body=None, **kwargs): class MockS3Client: """Mock S3 client for integration testing""" + # Elasticsearch repository plugin information (matches AwsS3Client defaults) + ES_PLUGIN_NAME = "repository-s3" + ES_PLUGIN_DISPLAY_NAME = "S3" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" + STORAGE_TYPE = "S3 bucket" + ES_KEYSTORE_INSTRUCTIONS = "Configure AWS credentials in Elasticsearch keystore" + STORAGE_CREATION_HELP = "Check AWS credentials and permissions" + def __init__(self): self._buckets = {} self._objects = {} From d946d300bc8592168d67f56dbdd360bbfa7a67b7 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:07:11 -0500 Subject: [PATCH 10/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20correct=20error=20m?= =?UTF-8?q?essages=20for=20index=20and=20bucket=20deletion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove non-existent "deepfreeze DELETE index" command from help text - Add STORAGE_DELETE_CMD to each client class with provider-specific deletion commands (aws s3 rb, az storage container delete, gcloud storage rm) - Update bucket exists error to show correct provider-specific command --- .../deepfreeze_core/actions/setup.py | 16 +++++++++------- .../deepfreeze_core/aws_client.py | 3 +++ .../deepfreeze_core/azure_client.py | 3 +++ .../deepfreeze_core/gcp_client.py | 3 +++ tests/cli/test_integration.py | 1 + 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py index e4a7481..ec3ac04 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py @@ -143,10 +143,10 @@ def _check_preconditions(self) -> None: errors.append( { "issue": f"Status index [cyan]{STATUS_INDEX}[/cyan] already exists", - "solution": f"Delete the existing index before running setup:\n" - f" [yellow]deepfreeze --host DELETE index --name {STATUS_INDEX}[/yellow]\n" - f" or use the Elasticsearch API:\n" - f" [yellow]curl -X DELETE 'http://:9200/{STATUS_INDEX}'[/yellow]", + "solution": f"Delete the existing index using the Elasticsearch API:\n" + f" [yellow]curl -X DELETE 'https://:9200/{STATUS_INDEX}'[/yellow]\n\n" + "Or via Kibana Dev Tools:\n" + f" [yellow]DELETE /{STATUS_INDEX}[/yellow]", } ) @@ -179,11 +179,13 @@ def _check_preconditions(self) -> None: # Third, check if the bucket already exists self.loggit.debug("Checking if bucket %s exists", self.new_bucket_name) if self.s3.bucket_exists(self.new_bucket_name): + storage_type = self.s3.STORAGE_TYPE + delete_cmd = self.s3.STORAGE_DELETE_CMD.format(bucket=self.new_bucket_name) errors.append( { - "issue": f"S3 bucket [cyan]{self.new_bucket_name}[/cyan] already exists", - "solution": f"Delete the existing bucket before running setup:\n" - f" [yellow]aws s3 rb s3://{self.new_bucket_name} --force[/yellow]\n" + "issue": f"{storage_type} [cyan]{self.new_bucket_name}[/cyan] already exists", + "solution": f"Delete the existing {storage_type.lower()} before running setup:\n" + f" [yellow]{delete_cmd}[/yellow]\n" "\n[bold]WARNING:[/bold] This will delete all data in the bucket!\n" "Or use a different bucket_name_prefix in your configuration.", } diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py index 546c0ad..adbbb8c 100644 --- a/packages/deepfreeze-core/deepfreeze_core/aws_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -54,6 +54,9 @@ class AwsS3Client(S3Client): - Verify AWS region settings - Check AWS account limits for S3 buckets""" + # Storage bucket deletion command template ({bucket} will be replaced) + STORAGE_DELETE_CMD = "aws s3 rb s3://{bucket} --force" + def __init__( self, region: str = None, diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py index 6684344..bf1abda 100644 --- a/packages/deepfreeze-core/deepfreeze_core/azure_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -61,6 +61,9 @@ class AzureBlobClient(S3Client): - Check if container name is valid (lowercase, no underscores, 3-63 chars) - Verify Azure RBAC permissions allow container creation""" + # Storage bucket deletion command template ({bucket} will be replaced) + STORAGE_DELETE_CMD = "az storage container delete --name {bucket} --account-name " + def __init__( self, connection_string: str = None, diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py index e5ae4a3..a43e8ac 100644 --- a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -56,6 +56,9 @@ class GcpStorageClient(S3Client): - Check if bucket name is globally unique - Verify IAM permissions allow storage.buckets.create""" + # Storage bucket deletion command template ({bucket} will be replaced) + STORAGE_DELETE_CMD = "gcloud storage rm -r gs://{bucket}" + def __init__( self, project: str = None, diff --git a/tests/cli/test_integration.py b/tests/cli/test_integration.py index 7eeec21..496ffbb 100644 --- a/tests/cli/test_integration.py +++ b/tests/cli/test_integration.py @@ -119,6 +119,7 @@ class MockS3Client: STORAGE_TYPE = "S3 bucket" ES_KEYSTORE_INSTRUCTIONS = "Configure AWS credentials in Elasticsearch keystore" STORAGE_CREATION_HELP = "Check AWS credentials and permissions" + STORAGE_DELETE_CMD = "aws s3 rb s3://{bucket} --force" def __init__(self): self._buckets = {} From a11ea814aae0ad81e708043f141f49e4384a9f1f Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:09:37 -0500 Subject: [PATCH 11/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20make=20unexpected?= =?UTF-8?q?=20error=20message=20provider-aware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change "AWS credentials" to use the provider display name from the storage client (e.g., "Azure credentials" for Azure provider). --- packages/deepfreeze-core/deepfreeze_core/actions/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py index ec3ac04..d856481 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py @@ -741,13 +741,14 @@ def do_action(self) -> None: if self.porcelain: print(f"ERROR\tunexpected\t{str(e)}") else: + provider_name = self.s3.ES_PLUGIN_DISPLAY_NAME self.console.print( Panel( f"[bold]An unexpected error occurred during setup[/bold]\n\n" f"Error: {escape(str(e))}\n\n" f"[bold]What to do:[/bold]\n" f" - Check the logs for detailed error information\n" - f" - Verify all prerequisites are met (AWS credentials, ES connection, etc.)\n" + f" - Verify all prerequisites are met ({provider_name} credentials, ES connection, etc.)\n" f" - You may need to manually clean up any partially created resources\n" f" - Run [yellow]deepfreeze cleanup[/yellow] to remove any partial state", title="[bold red]Unexpected Setup Error[/bold red]", From f6ae4e39d38d3d18e165d238c9678690b401ffae Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:35:31 -0500 Subject: [PATCH 12/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20support=20Azure=20a?= =?UTF-8?q?nd=20GCP=20repository=20types=20in=20create=5Frepo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The create_repo function was hardcoded to create S3 repositories. Updated to support: - AWS: type "s3" with bucket, base_path, canned_acl, storage_class - Azure: type "azure" with container, base_path - GCP: type "gcs" with bucket, base_path This was the root cause of Azure repository creation failures. --- .../deepfreeze_core/actions/rotate.py | 1 + .../deepfreeze_core/actions/setup.py | 2 + .../deepfreeze_core/utilities.py | 45 ++++++++++++++----- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index 5017abc..0e56fa1 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -144,6 +144,7 @@ def _create_new_repository(self, dry_run: bool = False) -> tuple: base_path, self.settings.canned_acl, self.settings.storage_class, + provider=self.settings.provider, ) # Update last_suffix in settings diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py index d856481..31b9349 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py @@ -386,6 +386,7 @@ def do_dry_run(self) -> None: self.base_path, self.settings.canned_acl, self.settings.storage_class, + provider=self.settings.provider, dry_run=True, ) @@ -485,6 +486,7 @@ def do_action(self) -> None: self.base_path, self.settings.canned_acl, self.settings.storage_class, + provider=self.settings.provider, ) self.loggit.info( "Successfully created repository %s", self.new_repo_name diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index 2fc28cc..c276784 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -317,43 +317,64 @@ def create_repo( base_path: str, canned_acl: str, storage_class: str, + provider: str = "aws", dry_run: bool = False, ) -> None: """ - Creates a new repo using the previously-created bucket. + Creates a new repo using the previously-created bucket/container. :param client: A client connection object :type client: Elasticsearch :param repo_name: The name of the repository to create :type repo_name: str - :param bucket_name: The name of the bucket to use for the repository + :param bucket_name: The name of the bucket/container to use for the repository :type bucket_name: str :param base_path: Path within a bucket where snapshots are stored :type base_path: str - :param canned_acl: One of the AWS canned ACL values + :param canned_acl: One of the AWS canned ACL values (AWS only) :type canned_acl: str - :param storage_class: AWS Storage class + :param storage_class: Storage class (AWS only) :type storage_class: str + :param provider: Cloud provider (aws, azure, gcp) + :type provider: str :param dry_run: If True, do not actually create the repository :type dry_run: bool :raises ActionError: If the repository cannot be created """ loggit = logging.getLogger("deepfreeze.utilities") - loggit.info("Creating repo %s using bucket %s", repo_name, bucket_name) + loggit.info("Creating repo %s using bucket %s (provider: %s)", repo_name, bucket_name, provider) if dry_run: return + + # Build repository settings based on provider + if provider == "azure": + repo_type = "azure" + settings = { + "container": bucket_name, + "base_path": base_path, + } + elif provider == "gcp": + repo_type = "gcs" + settings = { + "bucket": bucket_name, + "base_path": base_path, + } + else: # aws + repo_type = "s3" + settings = { + "bucket": bucket_name, + "base_path": base_path, + "canned_acl": canned_acl, + "storage_class": storage_class, + } + try: client.snapshot.create_repository( name=repo_name, body={ - "type": "s3", - "settings": { - "bucket": bucket_name, - "base_path": base_path, - "canned_acl": canned_acl, - "storage_class": storage_class, - }, + "type": repo_type, + "settings": settings, }, ) except Exception as e: From 5a84fa5c9110d0f31c763bbb9bc21502a8d7dfce Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:39:42 -0500 Subject: [PATCH 13/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20include=20storage?= =?UTF-8?q?=20account=20name=20in=20Azure=20error=20messages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add storage account name tracking to AzureBlobClient to help diagnose credential configuration issues. Error messages now show which account is being used when containers already exist or creation fails. Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze_core/azure_client.py | 100 +++++++++++++++--- 1 file changed, 87 insertions(+), 13 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py index bf1abda..0f06943 100644 --- a/packages/deepfreeze-core/deepfreeze_core/azure_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -71,6 +71,7 @@ def __init__( account_key: str = None, ) -> None: self.loggit = logging.getLogger("deepfreeze.azure_client") + self.account_name = None # Will be set during initialization try: # Priority: constructor args > environment variables conn_str = connection_string or os.environ.get( @@ -81,18 +82,23 @@ def __init__( if conn_str: self.service_client = BlobServiceClient.from_connection_string(conn_str) + # Extract account name from connection string + self.account_name = self._extract_account_name_from_conn_str(conn_str) self.loggit.debug( - "Using connection string for auth (source: %s)", + "Using connection string for auth (source: %s, account: %s)", "config" if connection_string else "environment", + self.account_name, ) elif acct_name and acct_key: account_url = f"https://{acct_name}.blob.core.windows.net" self.service_client = BlobServiceClient( account_url=account_url, credential=acct_key ) + self.account_name = acct_name self.loggit.debug( - "Using account name + key for auth (source: %s)", + "Using account name + key for auth (source: %s, account: %s)", "config" if account_name else "environment", + self.account_name, ) else: raise ActionError( @@ -137,33 +143,71 @@ def test_connection(self) -> bool: def create_bucket(self, bucket_name: str) -> None: """Create an Azure Blob container (equivalent to S3 bucket).""" - self.loggit.info(f"Creating container: {bucket_name}") + self.loggit.info( + "Creating container: %s in storage account: %s", + bucket_name, + self.account_name, + ) if self.bucket_exists(bucket_name): - self.loggit.info(f"Container {bucket_name} already exists") - raise ActionError(f"Container {bucket_name} already exists") + self.loggit.info( + "Container %s already exists in storage account %s", + bucket_name, + self.account_name, + ) + raise ActionError( + f"Container {bucket_name} already exists in storage account {self.account_name}" + ) try: self.service_client.create_container(bucket_name) - self.loggit.info(f"Successfully created container {bucket_name}") + self.loggit.info( + "Successfully created container %s in storage account %s", + bucket_name, + self.account_name, + ) except ResourceExistsError as e: - raise ActionError(f"Container {bucket_name} already exists") from e + raise ActionError( + f"Container {bucket_name} already exists in storage account {self.account_name}" + ) from e except AzureError as e: - self.loggit.error(f"Error creating container {bucket_name}: {e}") - raise ActionError(f"Error creating container {bucket_name}: {e}") from e + self.loggit.error( + "Error creating container %s in storage account %s: %s", + bucket_name, + self.account_name, + e, + ) + raise ActionError( + f"Error creating container {bucket_name} in storage account {self.account_name}: {e}" + ) from e def bucket_exists(self, bucket_name: str) -> bool: """Check if an Azure Blob container exists.""" - self.loggit.debug(f"Checking if container {bucket_name} exists") + self.loggit.debug( + "Checking if container %s exists in storage account %s", + bucket_name, + self.account_name, + ) try: container_client = self.service_client.get_container_client(bucket_name) container_client.get_container_properties() - self.loggit.debug(f"Container {bucket_name} exists") + self.loggit.debug( + "Container %s exists in storage account %s", + bucket_name, + self.account_name, + ) return True except ResourceNotFoundError: - self.loggit.debug(f"Container {bucket_name} does not exist") + self.loggit.debug( + "Container %s does not exist in storage account %s", + bucket_name, + self.account_name, + ) return False except AzureError as e: self.loggit.error( - "Error checking container existence for %s: %s", bucket_name, e + "Error checking container existence for %s in storage account %s: %s", + bucket_name, + self.account_name, + e, ) raise ActionError(e) from e @@ -580,3 +624,33 @@ def _format_restore_header(self, properties) -> str: # Was rehydrated from archive return 'ongoing-request="false"' return None + + def _extract_account_name_from_conn_str(self, conn_str: str) -> str: + """ + Extract the account name from an Azure connection string. + + Args: + conn_str (str): The Azure Storage connection string. + + Returns: + str: The storage account name, or "unknown" if not found. + """ + try: + for part in conn_str.split(";"): + if part.startswith("AccountName="): + return part.split("=", 1)[1] + except Exception: + pass + return "unknown" + + def get_storage_delete_cmd(self, bucket_name: str) -> str: + """ + Get the provider-specific command to delete a storage container. + + Args: + bucket_name (str): The name of the container to delete. + + Returns: + str: The command string with bucket and account name filled in. + """ + return f"az storage container delete --name {bucket_name} --account-name {self.account_name}" From 1890d91700a798271f9066e58db6d9c7e0fc8731 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 21 Jan 2026 11:45:57 -0500 Subject: [PATCH 14/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20remove=20hardcoded?= =?UTF-8?q?=20AWS/S3=20references=20throughout=20codebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix mount_repo() to use provider from settings for correct repo type (s3/azure/gcs) and settings (bucket vs container) - Update CLI defaults to validate all providers (aws, azure, gcp) - Update rotate.py log messages to use provider-specific storage type - Update constants.py comments to remove S3-specific terminology Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze/defaults/__init__.py | 2 +- .../deepfreeze_core/actions/rotate.py | 12 ++++-- .../deepfreeze_core/constants.py | 8 ++-- .../deepfreeze_core/utilities.py | 41 +++++++++++++++---- 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py b/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py index 0179fae..5f0e8ad 100644 --- a/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py +++ b/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py @@ -125,7 +125,7 @@ def provider(): """ Cloud provider for deepfreeze. """ - return {Optional("provider", default="aws"): Any("aws")} + return {Optional("provider", default="aws"): Any("aws", "azure", "gcp")} def rotate_by(): diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index 0e56fa1..f9c9891 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -120,21 +120,25 @@ def _create_new_repository(self, dry_run: bool = False) -> tuple: new_bucket_name = self.settings.bucket_name_prefix base_path = f"{self.settings.base_path_prefix}-{next_suffix}" + # Get provider-specific storage type for logging + storage_type = getattr(self.s3, "STORAGE_TYPE", "bucket").lower() + self.loggit.info( - "Creating new repository %s (bucket: %s, base_path: %s)", + "Creating new repository %s (%s: %s, base_path: %s)", new_repo_name, + storage_type, new_bucket_name, base_path, ) if not dry_run: - # Create bucket if rotating by bucket + # Create storage container/bucket if rotating by bucket if self.settings.rotate_by == "bucket": if not self.s3.bucket_exists(new_bucket_name): self.s3.create_bucket(new_bucket_name) - self.loggit.info("Created S3 bucket %s", new_bucket_name) + self.loggit.info("Created %s %s", storage_type, new_bucket_name) else: - self.loggit.info("S3 bucket %s already exists", new_bucket_name) + self.loggit.info("%s %s already exists", storage_type.capitalize(), new_bucket_name) # Create repository in Elasticsearch create_repo( diff --git a/packages/deepfreeze-core/deepfreeze_core/constants.py b/packages/deepfreeze-core/deepfreeze_core/constants.py index 248b5e5..5570d16 100644 --- a/packages/deepfreeze-core/deepfreeze_core/constants.py +++ b/packages/deepfreeze-core/deepfreeze_core/constants.py @@ -13,11 +13,11 @@ # Repository thaw lifecycle states THAW_STATE_ACTIVE = "active" # Active repository, never been through thaw lifecycle -THAW_STATE_FROZEN = "frozen" # In cold storage (Glacier), not currently accessible -THAW_STATE_THAWING = "thawing" # S3 restore in progress, waiting for retrieval -THAW_STATE_THAWED = "thawed" # S3 restore complete, mounted and in use +THAW_STATE_FROZEN = "frozen" # In cold storage (archive tier), not currently accessible +THAW_STATE_THAWING = "thawing" # Restore in progress, waiting for retrieval +THAW_STATE_THAWED = "thawed" # Restore complete, mounted and in use THAW_STATE_EXPIRED = ( - "expired" # S3 restore expired, reverted to Glacier, ready for cleanup + "expired" # Restore expired, reverted to archive tier, ready for cleanup ) THAW_STATES = [ diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index c276784..c2082ea 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -1322,21 +1322,46 @@ def mount_repo(client: Elasticsearch, repo: Repository) -> None: loggit = logging.getLogger("deepfreeze.utilities") loggit.info("Mounting repository %s", repo.name) - # Get settings to retrieve canned_acl and storage_class + # Get settings to retrieve provider and storage settings settings = get_settings(client) + provider = settings.provider + + # Build repository settings based on provider + if provider == "azure": + repo_type = "azure" + repo_settings = { + "container": repo.bucket, + "base_path": repo.base_path, + } + elif provider == "gcp": + repo_type = "gcs" + repo_settings = { + "bucket": repo.bucket, + "base_path": repo.base_path, + } + else: # aws (default) + repo_type = "s3" + repo_settings = { + "bucket": repo.bucket, + "base_path": repo.base_path, + "canned_acl": settings.canned_acl, + "storage_class": settings.storage_class, + } + + loggit.debug( + "Mounting repository %s with type=%s, settings=%s", + repo.name, + repo_type, + repo_settings, + ) # Create the repository in Elasticsearch try: client.snapshot.create_repository( name=repo.name, body={ - "type": "s3", - "settings": { - "bucket": repo.bucket, - "base_path": repo.base_path, - "canned_acl": settings.canned_acl, - "storage_class": settings.storage_class, - }, + "type": repo_type, + "settings": repo_settings, }, ) loggit.info("Repository %s created successfully", repo.name) From 81e8c64fd493a33e0c79eecc6262a7fca36b9d9c Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Thu, 22 Jan 2026 07:27:14 -0500 Subject: [PATCH 15/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20Azure=20archiving?= =?UTF-8?q?=20and=20add=20storage=20tier=20to=20status=20display?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix push_to_glacier to use refreeze() which handles provider-specific tier changes correctly (direct tier set for Azure/GCP vs copy-in-place for AWS) - Update all refreeze methods to skip objects already in target tier, avoiding errors when archiving already-archived blobs - Add Storage Tier column to status display showing actual blob tier (Archive, Hot, Cool, Mixed) to verify archiving is working Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze_core/actions/status.py | 80 +++++++++++++++- .../deepfreeze_core/aws_client.py | 14 ++- .../deepfreeze_core/azure_client.py | 26 ++++- .../deepfreeze_core/gcp_client.py | 17 +++- .../deepfreeze_core/utilities.py | 95 ++++++++----------- 5 files changed, 163 insertions(+), 69 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/status.py b/packages/deepfreeze-core/deepfreeze_core/actions/status.py index edb38b2..88b5b0a 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/status.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/status.py @@ -96,18 +96,78 @@ def _load_settings(self) -> None: # Initialize S3 client with provider from settings self.s3 = s3_client_factory(self.settings.provider) + def _get_repo_storage_tier(self, bucket: str, base_path: str) -> str: + """ + Sample blobs in a repository to determine the predominant storage tier. + + Returns a summary like "Archive", "Hot", "Cool", or "Mixed". + """ + try: + # Normalize path + path = base_path.strip("/") + if path: + path += "/" + + # List first few objects to sample storage tier + objects = self.s3.list_objects(bucket, path) + if not objects: + return "Empty" + + # Sample up to 10 objects + sample = objects[:10] + tiers = set() + for obj in sample: + # Get storage class from object metadata + storage_class = obj.get("StorageClass", "STANDARD") + # For Azure, also check BlobTier + blob_tier = obj.get("BlobTier") + if blob_tier: + tiers.add(blob_tier) + else: + tiers.add(storage_class) + + # Return summary + if len(tiers) == 1: + tier = tiers.pop() + # Normalize tier names for display + tier_display = { + "GLACIER": "Archive", + "DEEP_ARCHIVE": "Archive", + "Archive": "Archive", + "STANDARD": "Hot", + "Hot": "Hot", + "Cool": "Cool", + "STANDARD_IA": "Cool", + "ARCHIVE": "Archive", + "COLDLINE": "Archive", + "NEARLINE": "Cool", + }.get(tier, tier) + return tier_display + elif len(tiers) > 1: + return "Mixed" + else: + return "Unknown" + except Exception as e: + self.loggit.debug("Error getting storage tier for %s/%s: %s", bucket, base_path, e) + return "N/A" + def _get_repositories_status(self) -> list: """Get status of all repositories.""" repos = [] try: all_repos = get_all_repos(self.client) + # Get mounted repos list once + mounted_repos = get_matching_repo_names( + self.client, self.settings.repo_name_prefix + ) + for repo in all_repos: # Check if repo is actually mounted in ES - mounted_repos = get_matching_repo_names( - self.client, self.settings.repo_name_prefix - ) is_mounted_in_es = repo.name in mounted_repos + # Get storage tier (sample blobs) + storage_tier = self._get_repo_storage_tier(repo.bucket, repo.base_path) + repos.append( { "name": repo.name, @@ -117,6 +177,7 @@ def _get_repositories_status(self) -> list: "end": repo.end.isoformat() if repo.end else None, "is_mounted": is_mounted_in_es, "thaw_state": repo.thaw_state, + "storage_tier": storage_tier, "thawed_at": ( repo.thawed_at.isoformat() if repo.thawed_at else None ), @@ -258,6 +319,7 @@ def _display_rich( table.add_column("Date Range", style="white") table.add_column("Mounted", style="green") table.add_column("Thaw State", style="magenta") + table.add_column("Storage Tier", style="blue") for repo in sorted(display_repos, key=lambda x: x["name"]): date_range = "" @@ -278,6 +340,17 @@ def _display_rich( "expired": "red", }.get(thaw_state, "white") + # Storage tier with color coding + storage_tier = repo.get("storage_tier", "N/A") + tier_color = { + "Archive": "blue", + "Hot": "green", + "Cool": "cyan", + "Mixed": "yellow", + "Empty": "dim", + "N/A": "dim", + }.get(storage_tier, "white") + table.add_row( repo["name"], repo.get("bucket", "N/A"), @@ -285,6 +358,7 @@ def _display_rich( date_range, mounted_str, f"[{state_color}]{thaw_state}[/{state_color}]", + f"[{tier_color}]{storage_tier}[/{tier_color}]", ) self.console.print(table) diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py index adbbb8c..8a1a86c 100644 --- a/packages/deepfreeze-core/deepfreeze_core/aws_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -313,6 +313,7 @@ def refreeze( ) refrozen_count = 0 + skipped_count = 0 error_count = 0 paginator = self.client.get_paginator("list_objects_v2") @@ -329,6 +330,16 @@ def refreeze( key = obj["Key"] current_storage = obj.get("StorageClass", "STANDARD") + # Skip objects already in the target storage class + if current_storage == storage_class: + self.loggit.debug( + "Skipping object %s - already in %s", + key, + current_storage, + ) + skipped_count += 1 + continue + try: # Copy the object with a new storage class self.loggit.debug( @@ -360,8 +371,9 @@ def refreeze( # Log summary self.loggit.info( - "Refreeze operation completed - refrozen: %d, errors: %d", + "Refreeze operation completed - changed: %d, skipped (already archived): %d, errors: %d", refrozen_count, + skipped_count, error_count, ) diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py index 0f06943..f616e78 100644 --- a/packages/deepfreeze-core/deepfreeze_core/azure_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -349,27 +349,42 @@ def refreeze( container_client = self.service_client.get_container_client(bucket_name) refrozen_count = 0 + skipped_count = 0 error_count = 0 + # Map target tier to string for comparison + target_tier_str = str(target_tier).replace("StandardBlobTier.", "") + # List blobs with prefix blobs = container_client.list_blobs(name_starts_with=path) for blob in blobs: + current_tier = blob.blob_tier + + # Skip blobs already in the target tier + if current_tier == target_tier_str: + self.loggit.debug( + "Skipping blob %s - already in %s tier", + blob.name, + current_tier, + ) + skipped_count += 1 + continue + try: blob_client = container_client.get_blob_client(blob.name) - current_tier = blob.blob_tier self.loggit.debug( - "Refreezing blob: %s (from %s to %s)", + "Changing tier for blob: %s (from %s to %s)", blob.name, current_tier, - target_tier, + target_tier_str, ) blob_client.set_standard_blob_tier(target_tier) refrozen_count += 1 except AzureError as e: error_count += 1 self.loggit.error( - "Error refreezing blob %s: %s (type: %s)", + "Error changing tier for blob %s: %s (type: %s)", blob.name, str(e), type(e).__name__, @@ -377,8 +392,9 @@ def refreeze( ) self.loggit.info( - "Refreeze operation completed - refrozen: %d, errors: %d", + "Refreeze operation completed - changed: %d, skipped (already archived): %d, errors: %d", refrozen_count, + skipped_count, error_count, ) diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py index a43e8ac..4cbc1cc 100644 --- a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -288,14 +288,26 @@ def refreeze( bucket = self.client.bucket(bucket_name) refrozen_count = 0 + skipped_count = 0 error_count = 0 # List blobs with prefix blobs = bucket.list_blobs(prefix=path) for blob in blobs: + current_class = blob.storage_class + + # Skip blobs already in the target storage class + if current_class == target_class: + self.loggit.debug( + "Skipping blob %s - already in %s", + blob.name, + current_class, + ) + skipped_count += 1 + continue + try: - current_class = blob.storage_class self.loggit.debug( "Refreezing blob: %s (from %s to %s)", blob.name, @@ -315,8 +327,9 @@ def refreeze( ) self.loggit.info( - "Refreeze operation completed - refrozen: %d, errors: %d", + "Refreeze operation completed - changed: %d, skipped (already archived): %d, errors: %d", refrozen_count, + skipped_count, error_count, ) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index c2082ea..67e1c17 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -24,79 +24,58 @@ from deepfreeze_core.s3client import S3Client -def push_to_glacier(s3: S3Client, repo: Repository) -> bool: - """Push objects to Glacier storage +def push_to_glacier(s3: S3Client, repo: Repository, storage_class: str = "GLACIER") -> bool: + """Push objects to archive storage (Glacier for AWS, Archive tier for Azure, etc.) - :param s3: The S3 client object + Uses the storage client's refreeze method which handles provider-specific + archiving correctly: + - AWS S3: Copy-in-place with new storage class + - Azure: Direct tier change with set_standard_blob_tier() + - GCP: Direct storage class change + + :param s3: The storage client object :type s3: S3Client - :param repo: The repository to push to Glacier + :param repo: The repository to archive :type repo: Repository + :param storage_class: Target storage class (default: GLACIER) + :type storage_class: str - :return: True if all objects were successfully moved, False otherwise + :return: True if archiving completed (may have partial failures), False on error :rtype: bool - - :raises Exception: If the object is not in the restoration process """ + loggit = logging.getLogger("deepfreeze.utilities") + try: - # Normalize base_path: remove leading/trailing slashes, ensure it ends with / + # Normalize base_path: remove leading/trailing slashes base_path = repo.base_path.strip("/") if base_path: base_path += "/" - # Initialize variables for pagination - success = True - object_count = 0 - - # List objects - objects = s3.list_objects(repo.bucket, base_path) - - # Process each object - for obj in objects: - key = obj["Key"] - current_storage_class = obj.get("StorageClass", "STANDARD") - - # Log the object being processed - logging.info( - "Processing object: s3://%s/%s (Current: %s)", - repo.bucket, - key, - current_storage_class, - ) - - try: - # Copy object to itself with new storage class - copy_source = {"Bucket": repo.bucket, "Key": key} - s3.copy_object( - Bucket=repo.bucket, - Key=key, - CopySource=copy_source, - StorageClass="GLACIER", - ) - - # Log success - logging.info( - "Successfully moved s3://%s/%s to GLACIER", repo.bucket, key - ) - object_count += 1 + loggit.info( + "Archiving repository %s (bucket: %s, path: %s) to %s", + repo.name, + repo.bucket, + base_path, + storage_class, + ) - except botocore.exceptions.ClientError as e: - logging.error("Failed to move s3://%s/%s: %s", repo.bucket, key, e) - success = False - continue + # Use the storage client's refreeze method which handles provider-specific + # archiving correctly (direct tier change for Azure/GCP, copy-in-place for AWS) + s3.refreeze(repo.bucket, base_path, storage_class) - # Log summary - logging.info( - "Processed %d objects in s3://%s/%s", object_count, repo.bucket, base_path + loggit.info( + "Archive operation completed for repository %s", + repo.name, ) - if success: - logging.info("All objects successfully moved to GLACIER") - else: - logging.warning("Some objects failed to move to GLACIER") - - return success + return True - except botocore.exceptions.ClientError as e: - logging.error("Failed to process bucket s3://%s: %s", repo.bucket, e) + except Exception as e: + loggit.error( + "Failed to archive repository %s: %s", + repo.name, + e, + exc_info=True, + ) return False From f21213076eb126eadab81dc3ba888ac5f1bdfe1d Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Thu, 22 Jan 2026 07:52:01 -0500 Subject: [PATCH 16/30] =?UTF-8?q?=E2=9C=A8=20feat:=20delete=20orphaned=20I?= =?UTF-8?q?LM=20policies=20during=20rotation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add _cleanup_orphaned_policies() to rotate action that deletes ILM policies referencing repositories that have been unmounted. This runs automatically after archiving repos, so old versioned policies are cleaned up as part of the rotation process rather than requiring a separate cleanup run. Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze_core/actions/rotate.py | 113 +++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index f9c9891..1dfc842 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -23,8 +23,10 @@ get_ilm_policy, get_index_templates, get_matching_repos, + get_matching_repo_names, get_next_suffix, get_settings, + is_policy_safe_to_delete, push_to_glacier, save_settings, unmount_repo, @@ -364,6 +366,76 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: return archived_repos + def _cleanup_orphaned_policies(self, dry_run: bool = False) -> list: + """ + Delete ILM policies that reference unmounted deepfreeze repositories. + + This is called after archiving repos to clean up the old versioned + policies that now reference non-existent repositories. + + :param dry_run: If True, don't actually delete anything + :return: List of deleted policy names + """ + deleted_policies = [] + + if not self.settings.ilm_policy_name: + return deleted_policies + + try: + all_policies = self.client.ilm.get_lifecycle() + + # Get currently mounted repos + mounted_repos = set(get_matching_repo_names( + self.client, self.settings.repo_name_prefix + )) + + for policy_name, policy_data in all_policies.items(): + # Only check versioned policies matching our prefix + if not policy_name.startswith(f"{self.settings.ilm_policy_name}-"): + continue + + # Check if policy references a deepfreeze repo + policy_body = policy_data.get("policy", {}) + phases = policy_body.get("phases", {}) + + for _phase_name, phase_config in phases.items(): + actions = phase_config.get("actions", {}) + if "searchable_snapshot" in actions: + snapshot_repo = actions["searchable_snapshot"].get( + "snapshot_repository" + ) + # Check if it references a repo that's no longer mounted + if ( + snapshot_repo + and snapshot_repo.startswith(self.settings.repo_name_prefix) + and snapshot_repo not in mounted_repos + ): + # Check if policy is safe to delete + if is_policy_safe_to_delete(self.client, policy_name): + self.loggit.info( + "Deleting orphaned ILM policy %s (references unmounted repo %s)", + policy_name, + snapshot_repo, + ) + if not dry_run: + try: + self.client.ilm.delete_lifecycle(name=policy_name) + deleted_policies.append(policy_name) + except Exception as e: + self.loggit.error( + "Failed to delete policy %s: %s", + policy_name, + e, + ) + else: + deleted_policies.append(policy_name) + break + + except Exception as e: + self.loggit.warning("Error cleaning up orphaned policies: %s", e) + + return deleted_policies + def do_dry_run(self) -> None: """ Perform a dry-run of the rotation. @@ -434,6 +506,25 @@ def do_dry_run(self) -> None: ) ) + # Show orphaned policies that would be deleted + orphaned_policies = self._cleanup_orphaned_policies(dry_run=True) + if orphaned_policies: + if self.porcelain: + for policy in orphaned_policies: + print(f"DRY_RUN\tdelete_policy\t{policy}") + else: + policy_list = "\n".join( + [f" - [red]{p}[/red]" for p in orphaned_policies] + ) + self.console.print( + Panel( + f"[bold]Would delete {len(orphaned_policies)} orphaned ILM policies:[/bold]\n{policy_list}", + title="[bold blue]Dry Run - Delete Orphaned Policies[/bold blue]", + border_style="blue", + expand=False, + ) + ) + except (MissingIndexError, MissingSettingsError) as e: if self.porcelain: print(f"ERROR\t{type(e).__name__}\t{str(e)}") @@ -512,6 +603,25 @@ def do_action(self) -> None: ) ) + # Clean up orphaned ILM policies (policies referencing unmounted repos) + deleted_policies = self._cleanup_orphaned_policies() + if deleted_policies: + if self.porcelain: + for policy in deleted_policies: + print(f"DELETED\tilm_policy\t{policy}") + else: + policy_list = "\n".join( + [f" - [red]{p}[/red]" for p in deleted_policies] + ) + self.console.print( + Panel( + f"[bold]Deleted {len(deleted_policies)} orphaned ILM policies:[/bold]\n{policy_list}", + title="[bold green]Orphaned Policies Cleaned Up[/bold green]", + border_style="green", + expand=False, + ) + ) + # Final summary if not self.porcelain: self.console.print( @@ -519,7 +629,8 @@ def do_action(self) -> None: f"[bold green]Rotation completed successfully![/bold green]\n\n" f"New repository: [cyan]{new_repo}[/cyan]\n" f"Policies updated: {len(updated_policies)}\n" - f"Repositories archived: {len(archived)}\n\n" + f"Repositories archived: {len(archived)}\n" + f"Orphaned policies deleted: {len(deleted_policies)}\n\n" f"[bold]Next steps:[/bold]\n" f" - Verify ILM policies are using the new repository\n" f" - Monitor searchable snapshot transitions\n" From 0b3bebe4a48d41712c8829c6090a2d54cf46f81a Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Thu, 22 Jan 2026 08:25:29 -0500 Subject: [PATCH 17/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20unmount=5Frepo=20to?= =?UTF-8?q?=20handle=20Azure=20container=20setting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unmount_repo function was hardcoded to look for "bucket" in repository settings, but Azure repos use "container" instead. This caused unmounting to fail for Azure repos, preventing archiving during rotation. Co-Authored-By: Claude Opus 4.5 --- packages/deepfreeze-core/deepfreeze_core/utilities.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index 67e1c17..c627f5b 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -545,7 +545,8 @@ def unmount_repo(client: Elasticsearch, repo: str) -> Repository: loggit = logging.getLogger("deepfreeze.utilities") # Get repository info from Elasticsearch repo_info = client.snapshot.get_repository(name=repo)[repo] - bucket = repo_info["settings"]["bucket"] + # Handle different providers: AWS/GCP use "bucket", Azure uses "container" + bucket = repo_info["settings"].get("bucket") or repo_info["settings"].get("container") base_path = repo_info["settings"]["base_path"] # Get repository object from status index From 015489c4e2e60450164e56df6213faf9ba8c1af0 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Thu, 22 Jan 2026 08:29:22 -0500 Subject: [PATCH 18/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20update=20date=20ran?= =?UTF-8?q?ge=20before=20archiving=20to=20archive=20tier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When blobs are moved to Archive tier, they become unreadable without rehydration. The date range update was happening during unmount (after archiving), so it failed because snapshot metadata couldn't be read. Now the date range is updated BEFORE pushing to archive tier, while the snapshot metadata is still readable. Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze_core/actions/rotate.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index 1dfc842..863ebe2 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -25,11 +25,13 @@ get_matching_repos, get_matching_repo_names, get_next_suffix, + get_repository, get_settings, is_policy_safe_to_delete, push_to_glacier, save_settings, unmount_repo, + update_repository_date_range, update_template_ilm_policy, ) @@ -343,10 +345,18 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: if not dry_run: try: - # Push all objects to Glacier + # Update date range BEFORE archiving (snapshot metadata won't be + # readable after blobs are moved to archive tier) + repo_obj = get_repository(self.client, repo.name) + if update_repository_date_range(self.client, repo_obj): + self.loggit.info( + "Updated date range for %s before archiving", repo.name + ) + + # Push all objects to archive tier push_to_glacier(self.s3, repo) - # Unmount the repository + # Unmount the repository (skip date range update since we did it above) unmounted_repo = unmount_repo(self.client, repo.name) # Update thaw state to frozen From c42e4ae7669ca7b32a320e15cab7fdac3e5509d5 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Thu, 22 Jan 2026 09:18:40 -0500 Subject: [PATCH 19/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20check=20for=20activ?= =?UTF-8?q?e=20indices=20before=20archiving=20repos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes for the archive/unmount logic: 1. Don't persist "unmounted" state if ES unmount actually failed (was causing mounted=Yes + thaw_state=frozen inconsistency) 2. Check for active searchable snapshot indices BEFORE archiving (prevents archiving blobs then failing to unmount, leaving repo in broken state where ES can't read archived blobs) Repos with active indices are now skipped and will be retried on the next rotation after ILM deletes those indices. Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze_core/actions/rotate.py | 25 +++++- .../deepfreeze_core/utilities.py | 79 +++++++++++++++++-- 2 files changed, 96 insertions(+), 8 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index 863ebe2..4e529e2 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -29,6 +29,7 @@ get_settings, is_policy_safe_to_delete, push_to_glacier, + repo_has_active_indices, save_settings, unmount_repo, update_repository_date_range, @@ -340,7 +341,22 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: # Keep the newest 'keep' repos mounted repos_to_archive = repos[: -self.keep] if len(repos) > self.keep else [] + skipped_repos = [] + for repo in repos_to_archive: + # Check if repo has active indices BEFORE archiving + # (can't unmount a repo with active searchable snapshot indices) + has_active, active_indices = repo_has_active_indices(self.client, repo.name) + if has_active: + self.loggit.warning( + "Skipping archive of %s - has %d active indices: %s", + repo.name, + len(active_indices), + active_indices[:3], + ) + skipped_repos.append(repo.name) + continue + self.loggit.info("Archiving repository %s to Glacier", repo.name) if not dry_run: @@ -356,7 +372,7 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: # Push all objects to archive tier push_to_glacier(self.s3, repo) - # Unmount the repository (skip date range update since we did it above) + # Unmount the repository unmounted_repo = unmount_repo(self.client, repo.name) # Update thaw state to frozen @@ -374,6 +390,13 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: else: archived_repos.append(repo.name) + if skipped_repos: + self.loggit.info( + "Skipped %d repos with active indices (will retry on next rotation): %s", + len(skipped_repos), + skipped_repos, + ) + return archived_repos def _cleanup_orphaned_policies(self, dry_run: bool = False) -> list: diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index c627f5b..dec0743 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -101,6 +101,71 @@ def get_all_indices_in_repo(client: Elasticsearch, repository: str) -> list: return list(indices) +def repo_has_active_indices(client: Elasticsearch, repo_name: str) -> tuple[bool, list]: + """ + Check if a repository has any active searchable snapshot indices using it. + + This is used to determine if a repository can be safely archived/unmounted. + Elasticsearch won't allow unmounting a repo that has active indices. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The repository name to check + :type repo_name: str + + :returns: Tuple of (has_active_indices, list_of_index_names) + :rtype: tuple[bool, list] + """ + loggit = logging.getLogger("deepfreeze.utilities") + active_indices = [] + + try: + # Get all indices and check which ones are searchable snapshots from this repo + all_indices = client.cat.indices(format="json") + + for idx in all_indices: + index_name = idx.get("index", "") + # Searchable snapshots have names like "partial---" + # or the index settings will reference the repository + if repo_name in index_name: + active_indices.append(index_name) + continue + + # Also check index settings for repository reference + try: + settings = client.indices.get_settings(index=index_name) + store_type = ( + settings.get(index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("store", {}) + .get("type") + ) + snapshot_repo = ( + settings.get(index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("store", {}) + .get("snapshot", {}) + .get("repository_name") + ) + if store_type == "snapshot" and snapshot_repo == repo_name: + active_indices.append(index_name) + except Exception: + pass # Index might not exist or be accessible + + except Exception as e: + loggit.warning("Error checking for active indices in repo %s: %s", repo_name, e) + + loggit.debug( + "Repository %s has %d active indices: %s", + repo_name, + len(active_indices), + active_indices[:5] if active_indices else [], + ) + return len(active_indices) > 0, active_indices + + def get_timestamp_range(client: Elasticsearch, indices: list) -> tuple: """ Retrieve the earliest and latest @timestamp values from the given indices. @@ -566,21 +631,21 @@ def unmount_repo(client: Elasticsearch, repo: str) -> Repository: repo_obj.end.isoformat() if repo_obj.end else "None", ) - # Mark repository as unmounted - repo_obj.unmount() - msg = f"Recording repository details as {repo_obj}" - loggit.debug(msg) - # Remove the repository from Elasticsearch - loggit.debug("Removing repo %s", repo) + loggit.debug("Removing repo %s from Elasticsearch", repo) try: client.snapshot.delete_repository(name=repo) except Exception as e: loggit.warning("Repository %s could not be unmounted due to %s", repo, e) loggit.warning("Another attempt will be made when rotate runs next") + # Don't mark as unmounted if the actual unmount failed + raise + + # Only mark as unmounted AFTER successful ES unmount + repo_obj.unmount() + loggit.debug("Recording repository details as %s", repo_obj) # Update the status index with final repository state - loggit.debug("Updating repo: %s", repo_obj) client.update(index=STATUS_INDEX, doc=repo_obj.to_dict(), id=repo_obj.docid) loggit.debug("Repo %s removed", repo) return repo_obj From e5d770c03e28aa6be3a01a8c04a27610e3773fe8 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Fri, 23 Jan 2026 05:39:14 -0500 Subject: [PATCH 20/30] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20date=20range=20?= =?UTF-8?q?repair=20to=20repair-metadata=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add _update_date_ranges method to RepairMetadata action that computes and persists date ranges for mounted repositories with missing dates. The date range for a repository is computed by querying @timestamp values from mounted searchable snapshot indices. This can only be done for mounted repos since unmounted repos cannot be queried. Changes: - Add _update_date_ranges() method to compute missing date ranges - Update do_dry_run() to report missing date ranges - Update do_action() to repair date ranges along with state discrepancies - Update test to expect get_all_repos called twice (state + date ranges) This fixes an issue where date ranges were missing from status display because update_repository_date_range was only called during archiving. Now users can run 'deepfreeze repair-metadata' to populate dates for mounted repos that predate the date range tracking feature. Co-Authored-By: Claude Opus 4.5 --- .../actions/repair_metadata.py | 254 +++++++++++++----- tests/cli/test_actions.py | 3 +- 2 files changed, 196 insertions(+), 61 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py index 3241d18..41a4f4a 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py @@ -22,6 +22,7 @@ from deepfreeze_core.utilities import ( get_all_repos, get_settings, + update_repository_date_range, ) @@ -290,6 +291,75 @@ def _repair_discrepancy(self, discrepancy: dict, dry_run: bool = False) -> dict: return result + def _update_date_ranges(self, dry_run: bool = False) -> list: + """ + Update date ranges for all mounted repositories. + + Date ranges can only be computed for mounted repositories since we need + to query @timestamp values from mounted indices. + + :param dry_run: If True, don't actually update anything + :return: List of result dictionaries + """ + results = [] + repos = get_all_repos(self.client) + + for repo in repos: + # Only process mounted repos (we can query their indices) + if not repo.is_mounted: + self.loggit.debug( + "Skipping date range update for unmounted repo %s", repo.name + ) + continue + + # Skip if dates already set + if repo.start and repo.end: + self.loggit.debug( + "Repo %s already has date range: %s to %s", + repo.name, + repo.start.isoformat(), + repo.end.isoformat(), + ) + continue + + result = { + "repo": repo.name, + "success": False, + "old_start": repo.start.isoformat() if repo.start else None, + "old_end": repo.end.isoformat() if repo.end else None, + "new_start": None, + "new_end": None, + "error": None, + } + + if dry_run: + # In dry run, report that we would update + result["success"] = True + results.append(result) + continue + + try: + # Use the utility function to update date range + if update_repository_date_range(self.client, repo): + result["success"] = True + result["new_start"] = repo.start.isoformat() if repo.start else None + result["new_end"] = repo.end.isoformat() if repo.end else None + self.loggit.info( + "Updated date range for %s: %s to %s", + repo.name, + result["new_start"], + result["new_end"], + ) + else: + result["error"] = "No timestamp data found in indices" + except Exception as e: + result["error"] = str(e) + self.loggit.error("Failed to update date range for %s: %s", repo.name, e) + + results.append(result) + + return results + def do_dry_run(self) -> None: """ Scan and report discrepancies without making changes. @@ -309,6 +379,9 @@ def do_dry_run(self) -> None: discrepancies = self._scan_repositories() + # Also check for missing date ranges on mounted repos + date_range_updates = self._update_date_ranges(dry_run=True) + if self.porcelain: for d in discrepancies: if d.get("error"): @@ -318,14 +391,16 @@ def do_dry_run(self) -> None: f"DISCREPANCY\t{d['repo']}\t{d['recorded_state']}\t" f"{d['actual_state']}\t{d.get('total_objects', 0)} objects" ) - print(f"SUMMARY\t{len(discrepancies)} discrepancies found") + for u in date_range_updates: + print(f"DATE_RANGE\t{u['repo']}\tmissing") + print(f"SUMMARY\t{len(discrepancies)} discrepancies\t{len(date_range_updates)} missing date ranges") return - if not discrepancies: + if not discrepancies and not date_range_updates: self.console.print( Panel( - "[green]No discrepancies found![/green]\n\n" - "All repository states in the status index match the actual S3 storage classes.", + "[green]No issues found![/green]\n\n" + "All repository states and date ranges are correct.", title="[bold green]Scan Complete[/bold green]", border_style="green", expand=False, @@ -365,13 +440,33 @@ def do_dry_run(self) -> None: storage_str, ) - self.console.print(table) - self.console.print() + if discrepancies: + self.console.print(table) + self.console.print() + + # Display date range updates table if any + if date_range_updates: + date_table = Table(title="Missing Date Ranges") + date_table.add_column("Repository", style="cyan") + date_table.add_column("Status", style="yellow") + + for u in date_range_updates: + date_table.add_row(u["repo"], "Missing - will compute from indices") + + self.console.print(date_table) + self.console.print() + + # Summary + summary_parts = [] + if discrepancies: + summary_parts.append(f"{len(discrepancies)} state discrepancies") + if date_range_updates: + summary_parts.append(f"{len(date_range_updates)} missing date ranges") self.console.print( Panel( - f"[bold]Found {len(discrepancies)} discrepancies[/bold]\n\n" - f"Run without [yellow]--dry-run[/yellow] to repair these discrepancies.", + f"[bold]Found {' and '.join(summary_parts)}[/bold]\n\n" + f"Run without [yellow]--dry-run[/yellow] to repair.", title="[bold blue]Dry Run Summary[/bold blue]", border_style="blue", expand=False, @@ -404,14 +499,17 @@ def do_action(self) -> None: discrepancies = self._scan_repositories() - if not discrepancies: + # Update date ranges for mounted repos (actual update, not dry run) + date_range_results = self._update_date_ranges(dry_run=False) + + if not discrepancies and not date_range_results: if self.porcelain: - print("COMPLETE\t0 discrepancies") + print("COMPLETE\t0 discrepancies\t0 date ranges") else: self.console.print( Panel( - "[green]No discrepancies found![/green]\n\n" - "All repository states in the status index match the actual S3 storage classes.", + "[green]No issues found![/green]\n\n" + "All repository states and date ranges are correct.", title="[bold green]Scan Complete[/bold green]", border_style="green", expand=False, @@ -419,68 +517,103 @@ def do_action(self) -> None: ) return - if not self.porcelain: - self.console.print( - f"[bold]Found {len(discrepancies)} discrepancies. Repairing...[/bold]" - ) - - # Repair each discrepancy - results = [] - for d in discrepancies: - result = self._repair_discrepancy(d) - results.append(result) + # Repair state discrepancies + state_results = [] + if discrepancies: + if not self.porcelain: + self.console.print( + f"[bold]Found {len(discrepancies)} state discrepancies. Repairing...[/bold]" + ) + for d in discrepancies: + result = self._repair_discrepancy(d) + state_results.append(result) # Display results + state_success = sum(1 for r in state_results if r["success"]) + state_fail = sum(1 for r in state_results if not r["success"]) + date_success = sum(1 for r in date_range_results if r["success"]) + date_fail = sum(1 for r in date_range_results if not r["success"]) + if self.porcelain: - for r in results: + for r in state_results: + status = "SUCCESS" if r["success"] else "FAILED" + print(f"STATE\t{status}\t{r['repo']}\t{r['old_state']}\t{r['new_state']}") + for r in date_range_results: status = "SUCCESS" if r["success"] else "FAILED" - print(f"{status}\t{r['repo']}\t{r['old_state']}\t{r['new_state']}") - success_count = sum(1 for r in results if r["success"]) - fail_count = sum(1 for r in results if not r["success"]) - print(f"COMPLETE\t{success_count} repaired\t{fail_count} failed") + error_msg = f"\t{r['error']}" if r.get("error") else "" + print(f"DATE_RANGE\t{status}\t{r['repo']}\t{r.get('new_start')}\t{r.get('new_end')}{error_msg}") + print(f"COMPLETE\t{state_success + date_success} repaired\t{state_fail + date_fail} failed") else: - # Summary - success_count = sum(1 for r in results if r["success"]) - fail_count = sum(1 for r in results if not r["success"]) - - if fail_count == 0: + total_fail = state_fail + date_fail + + if total_fail == 0: + summary_parts = [] + if state_success: + summary_parts.append(f"{state_success} state discrepancies") + if date_success: + summary_parts.append(f"{date_success} date ranges") self.console.print( Panel( - f"[bold green]Successfully repaired {success_count} discrepancies![/bold green]", + f"[bold green]Successfully repaired {' and '.join(summary_parts)}![/bold green]", title="[bold green]Repair Complete[/bold green]", border_style="green", expand=False, ) ) else: - # Show detailed results - table = Table(title="Repair Results") - table.add_column("Repository", style="cyan") - table.add_column("Old State", style="yellow") - table.add_column("New State", style="green") - table.add_column("Status", style="white") - - for r in results: - if r["success"]: - status = "[green]Repaired[/green]" - else: - status = f"[red]Failed: {r.get('error', 'Unknown')}[/red]" - - table.add_row( - r["repo"], - r.get("old_state", "N/A"), - r.get("new_state", "N/A"), - status, - ) - - self.console.print(table) - self.console.print() + # Show detailed results for state discrepancies + if state_results: + table = Table(title="State Repair Results") + table.add_column("Repository", style="cyan") + table.add_column("Old State", style="yellow") + table.add_column("New State", style="green") + table.add_column("Status", style="white") + + for r in state_results: + if r["success"]: + status = "[green]Repaired[/green]" + else: + status = f"[red]Failed: {r.get('error', 'Unknown')}[/red]" + + table.add_row( + r["repo"], + r.get("old_state", "N/A"), + r.get("new_state", "N/A"), + status, + ) + + self.console.print(table) + self.console.print() + + # Show detailed results for date range updates + if date_range_results: + date_table = Table(title="Date Range Repair Results") + date_table.add_column("Repository", style="cyan") + date_table.add_column("Start", style="green") + date_table.add_column("End", style="green") + date_table.add_column("Status", style="white") + + for r in date_range_results: + if r["success"]: + status = "[green]Updated[/green]" + else: + status = f"[red]Failed: {r.get('error', 'Unknown')}[/red]" + + date_table.add_row( + r["repo"], + r.get("new_start") or "N/A", + r.get("new_end") or "N/A", + status, + ) + + self.console.print(date_table) + self.console.print() self.console.print( Panel( f"[bold]Repair completed with some failures[/bold]\n\n" - f"Repaired: {success_count}\n" - f"Failed: {fail_count}\n\n" + f"State repairs: {state_success} succeeded, {state_fail} failed\n" + f"Date ranges: {date_success} updated, {date_fail} failed\n\n" f"Check logs for details on failures.", title="[bold yellow]Repair Complete[/bold yellow]", border_style="yellow", @@ -489,9 +622,10 @@ def do_action(self) -> None: ) self.loggit.info( - "RepairMetadata complete: %d repaired, %d failed", - success_count, - fail_count, + "RepairMetadata complete: %d state repairs, %d date ranges updated, %d failed", + state_success, + date_success, + state_fail + date_fail, ) except (MissingIndexError, MissingSettingsError) as e: diff --git a/tests/cli/test_actions.py b/tests/cli/test_actions.py index 281c46f..c02d43c 100644 --- a/tests/cli/test_actions.py +++ b/tests/cli/test_actions.py @@ -575,7 +575,8 @@ def test_repair_metadata_dry_run_scans_repos(self): repair = RepairMetadata(client=mock_client, porcelain=True) repair.do_dry_run() - mock_repos.assert_called_once() + # Called twice: once for state scan, once for date range update + assert mock_repos.call_count == 2 class TestActionInterfaceConsistency: From 102cd3c100da40c660b099442b0b7aff9a49b560 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Fri, 23 Jan 2026 05:46:18 -0500 Subject: [PATCH 21/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20don't=20treat=20act?= =?UTF-8?q?ive=20repos=20as=20thawed=20based=20on=20storage=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repair-metadata command was incorrectly flagging active repos in hot storage as "thawed" because both states use instant-access storage. The distinction between active (never archived) and thawed (restored from archive) is semantic, not storage-based. Fix the discrepancy detection to only flag actual contradictions: - active/thawed with archive storage → should be frozen - frozen with instant-access storage → should be thawed - thawing with all-archive storage → should be frozen - thawing with all-accessible storage → should be thawed This prevents false positives where active repos are incorrectly identified as needing repair. Co-Authored-By: Claude Opus 4.5 --- .../actions/repair_metadata.py | 55 +++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py index 41a4f4a..62f5a70 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py @@ -200,15 +200,62 @@ def _scan_repositories(self) -> list: ) continue - # Compare recorded state vs actual state - if repo.thaw_state != actual["determined_state"]: + # Compare recorded state vs actual storage state + # Note: "active" and "thawed" both use instant-access storage, so they're + # indistinguishable by storage class alone. We only flag a discrepancy + # when the storage state contradicts the recorded state. + is_discrepancy = False + suggested_state = None + + if repo.thaw_state in [THAW_STATE_ACTIVE, THAW_STATE_THAWED]: + # These states should have instant-access storage + if actual["glacier"] == actual["total_objects"] and actual["total_objects"] > 0: + # All objects in archive - should be frozen + is_discrepancy = True + suggested_state = THAW_STATE_FROZEN + elif actual["restoring"] > 0: + # Some objects restoring - should be thawing + is_discrepancy = True + suggested_state = THAW_STATE_THAWING + # If instant_access, no discrepancy - active/thawed both valid + + elif repo.thaw_state == THAW_STATE_FROZEN: + # This state should have archive storage + if actual["instant_access"] == actual["total_objects"] and actual["total_objects"] > 0: + # All objects accessible - was restored (thawed) + is_discrepancy = True + suggested_state = THAW_STATE_THAWED + elif actual["restoring"] > 0: + # Some objects restoring - thaw in progress + is_discrepancy = True + suggested_state = THAW_STATE_THAWING + + elif repo.thaw_state == THAW_STATE_THAWING: + # This state should have some objects restoring or all accessible + if actual["glacier"] == actual["total_objects"] and actual["total_objects"] > 0: + # All still in archive - thaw failed or not started + is_discrepancy = True + suggested_state = THAW_STATE_FROZEN + elif actual["instant_access"] == actual["total_objects"] and actual["total_objects"] > 0: + # All accessible - thaw completed + is_discrepancy = True + suggested_state = THAW_STATE_THAWED + + elif repo.thaw_state == THAW_STATE_EXPIRED: + # Expired should have archive storage (restore expired) + if actual["instant_access"] == actual["total_objects"] and actual["total_objects"] > 0: + # Still accessible - not actually expired + is_discrepancy = True + suggested_state = THAW_STATE_THAWED + + if is_discrepancy and suggested_state: discrepancies.append( { "repo": repo.name, "bucket": repo.bucket, "base_path": repo.base_path, "recorded_state": repo.thaw_state, - "actual_state": actual["determined_state"], + "actual_state": suggested_state, "storage_classes": actual["storage_classes"], "total_objects": actual["total_objects"], "instant_access": actual["instant_access"], @@ -220,7 +267,7 @@ def _scan_repositories(self) -> list: "Discrepancy found for %s: recorded=%s, actual=%s", repo.name, repo.thaw_state, - actual["determined_state"], + suggested_state, ) return discrepancies From be2a34421c7a0760d0ecd5b9ec1eb5956f55d13f Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Fri, 23 Jan 2026 05:48:16 -0500 Subject: [PATCH 22/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20add=20logging=20and?= =?UTF-8?q?=20ES=20verification=20to=20date=20range=20updates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add logging before each date range update attempt so we can see which repo the command is processing. Also verify that the repository actually exists in Elasticsearch before trying to query its snapshots, to avoid hanging on repos that are marked as mounted in the status index but aren't actually present in ES. Co-Authored-By: Claude Opus 4.5 --- .../deepfreeze_core/actions/repair_metadata.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py index 62f5a70..0e20f5c 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py @@ -369,6 +369,8 @@ def _update_date_ranges(self, dry_run: bool = False) -> list: ) continue + self.loggit.info("Checking date range for repo %s", repo.name) + result = { "repo": repo.name, "success": False, @@ -379,6 +381,21 @@ def _update_date_ranges(self, dry_run: bool = False) -> list: "error": None, } + # Verify the repo actually exists in ES before trying to query it + try: + es_repos = self.client.snapshot.get_repository(name=repo.name) + if repo.name not in es_repos: + self.loggit.debug( + "Repo %s marked as mounted but not found in ES, skipping", + repo.name, + ) + continue + except Exception as e: + self.loggit.debug( + "Repo %s not accessible in ES: %s, skipping", repo.name, e + ) + continue + if dry_run: # In dry run, report that we would update result["success"] = True From d2c7230bbe23e3c20b2a88a2181ec23a8d2fdb39 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Mon, 26 Jan 2026 16:25:48 -0500 Subject: [PATCH 23/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20raise=20ActionError?= =?UTF-8?q?=20on=20template=20PUT=20failure=20instead=20of=20silently=20sw?= =?UTF-8?q?allowing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Separate GET and PUT operations into distinct try blocks in update_index_template_ilm_policy() for both composable and legacy template paths. Previously, a PUT failure after a successful GET was caught by a generic except handler, logged at DEBUG level, and silently swallowed — causing the code to fall through and report the template as "not found" when it was actually found but failed to update. 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../deepfreeze_core/utilities.py | 176 ++++++++++-------- 1 file changed, 98 insertions(+), 78 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index dec0743..4abdabe 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -881,92 +881,71 @@ def update_index_template_ilm_policy( # First try composable templates (ES 7.8+) try: templates = client.indices.get_index_template(name=template_name) - if ( - templates - and "index_templates" in templates - and len(templates["index_templates"]) > 0 - ): - template_data = templates["index_templates"][0]["index_template"] - loggit.debug("Found composable template %s", template_name) - - # Ensure template structure exists - if "template" not in template_data: - template_data["template"] = {} - if "settings" not in template_data["template"]: - template_data["template"]["settings"] = {} - if "index" not in template_data["template"]["settings"]: - template_data["template"]["settings"]["index"] = {} - if "lifecycle" not in template_data["template"]["settings"]["index"]: - template_data["template"]["settings"]["index"]["lifecycle"] = {} - - # Get old policy name for logging - old_policy = template_data["template"]["settings"]["index"][ - "lifecycle" - ].get("name", "none") - - # Set the new ILM policy - template_data["template"]["settings"]["index"]["lifecycle"][ - "name" - ] = ilm_policy_name - - # Put the updated template - client.indices.put_index_template(name=template_name, body=template_data) - loggit.info( - "Updated composable template %s: ILM policy %s -> %s", - template_name, - old_policy, - ilm_policy_name, - ) - return { - "action": "updated", - "template_type": "composable", - "old_policy": old_policy, - "new_policy": ilm_policy_name, - } except NotFoundError: + templates = None loggit.debug( "Composable template %s not found, trying legacy template", template_name ) except Exception as e: + templates = None loggit.debug("Error checking composable template %s: %s", template_name, e) + if ( + templates + and "index_templates" in templates + and len(templates["index_templates"]) > 0 + ): + template_data = templates["index_templates"][0]["index_template"] + loggit.debug("Found composable template %s", template_name) + + # Ensure template structure exists + if "template" not in template_data: + template_data["template"] = {} + if "settings" not in template_data["template"]: + template_data["template"]["settings"] = {} + if "index" not in template_data["template"]["settings"]: + template_data["template"]["settings"]["index"] = {} + if "lifecycle" not in template_data["template"]["settings"]["index"]: + template_data["template"]["settings"]["index"]["lifecycle"] = {} + + # Get old policy name for logging + old_policy = template_data["template"]["settings"]["index"][ + "lifecycle" + ].get("name", "none") + + # Set the new ILM policy + template_data["template"]["settings"]["index"]["lifecycle"][ + "name" + ] = ilm_policy_name + + # Put the updated template + try: + client.indices.put_index_template( + name=template_name, body=template_data + ) + except Exception as e: + loggit.error( + "Error updating composable template %s: %s", template_name, e + ) + raise ActionError( + f"Failed to update composable template {template_name}: {e}" + ) from e + loggit.info( + "Updated composable template %s: ILM policy %s -> %s", + template_name, + old_policy, + ilm_policy_name, + ) + return { + "action": "updated", + "template_type": "composable", + "old_policy": old_policy, + "new_policy": ilm_policy_name, + } + # Try legacy templates try: templates = client.indices.get_template(name=template_name) - if templates and template_name in templates: - template_data = templates[template_name] - loggit.debug("Found legacy template %s", template_name) - - # Ensure template structure exists - if "settings" not in template_data: - template_data["settings"] = {} - if "index" not in template_data["settings"]: - template_data["settings"]["index"] = {} - if "lifecycle" not in template_data["settings"]["index"]: - template_data["settings"]["index"]["lifecycle"] = {} - - # Get old policy name for logging - old_policy = template_data["settings"]["index"]["lifecycle"].get( - "name", "none" - ) - - # Set the new ILM policy - template_data["settings"]["index"]["lifecycle"]["name"] = ilm_policy_name - - # Put the updated template - client.indices.put_template(name=template_name, body=template_data) - loggit.info( - "Updated legacy template %s: ILM policy %s -> %s", - template_name, - old_policy, - ilm_policy_name, - ) - return { - "action": "updated", - "template_type": "legacy", - "old_policy": old_policy, - "new_policy": ilm_policy_name, - } except NotFoundError: loggit.warning( "Template %s not found (checked both composable and legacy)", template_name @@ -977,8 +956,49 @@ def update_index_template_ilm_policy( "error": f"Template {template_name} not found", } except Exception as e: - loggit.error("Error updating legacy template %s: %s", template_name, e) - raise ActionError(f"Failed to update template {template_name}: {e}") from e + loggit.error("Error checking legacy template %s: %s", template_name, e) + raise ActionError(f"Failed to check legacy template {template_name}: {e}") from e + + if templates and template_name in templates: + template_data = templates[template_name] + loggit.debug("Found legacy template %s", template_name) + + # Ensure template structure exists + if "settings" not in template_data: + template_data["settings"] = {} + if "index" not in template_data["settings"]: + template_data["settings"]["index"] = {} + if "lifecycle" not in template_data["settings"]["index"]: + template_data["settings"]["index"]["lifecycle"] = {} + + # Get old policy name for logging + old_policy = template_data["settings"]["index"]["lifecycle"].get( + "name", "none" + ) + + # Set the new ILM policy + template_data["settings"]["index"]["lifecycle"]["name"] = ilm_policy_name + + # Put the updated template + try: + client.indices.put_template(name=template_name, body=template_data) + except Exception as e: + loggit.error("Error updating legacy template %s: %s", template_name, e) + raise ActionError( + f"Failed to update legacy template {template_name}: {e}" + ) from e + loggit.info( + "Updated legacy template %s: ILM policy %s -> %s", + template_name, + old_policy, + ilm_policy_name, + ) + return { + "action": "updated", + "template_type": "legacy", + "old_policy": old_policy, + "new_policy": ilm_policy_name, + } def create_thawed_ilm_policy(client: Elasticsearch, repo_name: str) -> str: From 2a19a4168853a93c5d61b803a3820ff03206a1c4 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Mon, 26 Jan 2026 16:35:59 -0500 Subject: [PATCH 24/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20strip=20system-mana?= =?UTF-8?q?ged=20created=5Fdate=20before=20PUT=20on=20index=20templates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Elasticsearch returns created_date in get_index_template responses but rejects it on put_index_template. Strip this field before both PUT call sites in update_index_template_ilm_policy and update_template_ilm_policy. 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- packages/deepfreeze-core/deepfreeze_core/utilities.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index 4abdabe..37d9da5 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -918,6 +918,9 @@ def update_index_template_ilm_policy( "name" ] = ilm_policy_name + # Remove system-managed fields that ES rejects on PUT + template_data.pop("created_date", None) + # Put the updated template try: client.indices.put_index_template( @@ -1740,6 +1743,9 @@ def update_template_ilm_policy( "name" ] = new_policy_name + # Remove system-managed fields that ES rejects on PUT + template.pop("created_date", None) + client.indices.put_index_template(name=template_name, body=template) loggit.info( "Updated composable template %s to use policy %s", From 55b8c669f458be89e324e4a8e3cc1d69e14d63ac Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Mon, 26 Jan 2026 17:06:22 -0500 Subject: [PATCH 25/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20whitelist=20PUT=20f?= =?UTF-8?q?ields=20for=20composable=20template=20updates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace blacklist approach (popping created_date) with a whitelist of fields accepted by put_index_template. This ensures no system-managed metadata from the GET response leaks into the PUT body. 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../deepfreeze_core/utilities.py | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index 37d9da5..de0d8fd 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -918,13 +918,26 @@ def update_index_template_ilm_policy( "name" ] = ilm_policy_name - # Remove system-managed fields that ES rejects on PUT - template_data.pop("created_date", None) + # Only include fields accepted by put_index_template to avoid + # sending system-managed fields (e.g. created_date) from the GET response + _COMPOSABLE_TEMPLATE_FIELDS = { + "index_patterns", + "template", + "composed_of", + "priority", + "version", + "_meta", + "data_stream", + "allow_auto_create", + "deprecated", + "ignore_missing_component_templates", + } + put_body = {k: v for k, v in template_data.items() if k in _COMPOSABLE_TEMPLATE_FIELDS} # Put the updated template try: client.indices.put_index_template( - name=template_name, body=template_data + name=template_name, body=put_body ) except Exception as e: loggit.error( @@ -1743,10 +1756,23 @@ def update_template_ilm_policy( "name" ] = new_policy_name - # Remove system-managed fields that ES rejects on PUT - template.pop("created_date", None) - - client.indices.put_index_template(name=template_name, body=template) + # Only include fields accepted by put_index_template to avoid + # sending system-managed fields (e.g. created_date) from the GET response + _COMPOSABLE_FIELDS = { + "index_patterns", + "template", + "composed_of", + "priority", + "version", + "_meta", + "data_stream", + "allow_auto_create", + "deprecated", + "ignore_missing_component_templates", + } + put_body = {k: v for k, v in template.items() if k in _COMPOSABLE_FIELDS} + + client.indices.put_index_template(name=template_name, body=put_body) loggit.info( "Updated composable template %s to use policy %s", template_name, From 844458b36f0086762c6f3a160bb9d0a52a3499ed Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Tue, 27 Jan 2026 09:54:51 -0500 Subject: [PATCH 26/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20capture=20date=20ra?= =?UTF-8?q?nges=20for=20all=20mounted=20repos=20during=20rotation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Date ranges were never populated because the only update call lived inside _archive_old_repos, which runs after searchable snapshot indices are already gone. Move date range capture to a dedicated _update_date_ranges() step that runs before archiving, while indices are still queryable. - Add _update_date_ranges() method to Rotate class - Call it in do_action() between ILM policy update and archive step - Remove now-redundant update_repository_date_range call from _archive_old_repos() - Remove unused get_repository import - Add test verifying _update_date_ranges behaviour Co-Authored-By: Claude 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) --- .../deepfreeze_core/actions/rotate.py | 60 +++++++++++++++---- tests/cli/test_actions.py | 47 +++++++++++++++ 2 files changed, 97 insertions(+), 10 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index 4e529e2..0b7b80d 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -22,10 +22,9 @@ get_composable_templates, get_ilm_policy, get_index_templates, - get_matching_repos, get_matching_repo_names, + get_matching_repos, get_next_suffix, - get_repository, get_settings, is_policy_safe_to_delete, push_to_glacier, @@ -361,14 +360,6 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: if not dry_run: try: - # Update date range BEFORE archiving (snapshot metadata won't be - # readable after blobs are moved to archive tier) - repo_obj = get_repository(self.client, repo.name) - if update_repository_date_range(self.client, repo_obj): - self.loggit.info( - "Updated date range for %s before archiving", repo.name - ) - # Push all objects to archive tier push_to_glacier(self.s3, repo) @@ -399,6 +390,34 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: return archived_repos + def _update_date_ranges(self) -> list: + """ + Update date ranges for all mounted repositories. + + Queries each mounted repo's indices to capture min/max @timestamp values. + This must run while repos still have searchable snapshot indices, i.e. + before the archive step unmounts them. + + :return: List of repo names whose date ranges were updated + """ + updated_repos = [] + + repos = get_matching_repos( + self.client, self.settings.repo_name_prefix, mounted=True + ) + + for repo in repos: + try: + if update_repository_date_range(self.client, repo): + self.loggit.info("Updated date range for %s", repo.name) + updated_repos.append(repo.name) + except Exception as e: + self.loggit.error( + "Failed to update date range for %s: %s", repo.name, e + ) + + return updated_repos + def _cleanup_orphaned_policies(self, dry_run: bool = False) -> list: """ Delete ILM policies that reference unmounted deepfreeze repositories. @@ -617,6 +636,26 @@ def do_action(self) -> None: ) ) + # Update date ranges for all mounted repos (before archiving + # removes searchable snapshot indices) + updated_date_ranges = self._update_date_ranges() + if updated_date_ranges: + if self.porcelain: + for repo in updated_date_ranges: + print(f"UPDATED\tdate_range\t{repo}") + else: + range_list = "\n".join( + [f" - [cyan]{r}[/cyan]" for r in updated_date_ranges] + ) + self.console.print( + Panel( + f"[bold]Updated date ranges for {len(updated_date_ranges)} repositories:[/bold]\n{range_list}", + title="[bold green]Date Ranges Updated[/bold green]", + border_style="green", + expand=False, + ) + ) + # Archive old repositories archived = self._archive_old_repos() if archived: @@ -662,6 +701,7 @@ def do_action(self) -> None: f"[bold green]Rotation completed successfully![/bold green]\n\n" f"New repository: [cyan]{new_repo}[/cyan]\n" f"Policies updated: {len(updated_policies)}\n" + f"Date ranges updated: {len(updated_date_ranges)}\n" f"Repositories archived: {len(archived)}\n" f"Orphaned policies deleted: {len(deleted_policies)}\n\n" f"[bold]Next steps:[/bold]\n" diff --git a/tests/cli/test_actions.py b/tests/cli/test_actions.py index c02d43c..ca01e15 100644 --- a/tests/cli/test_actions.py +++ b/tests/cli/test_actions.py @@ -282,6 +282,53 @@ def test_rotate_calculates_next_suffix(self): assert new_repo == "deepfreeze-000006" + def test_update_date_ranges_calls_for_each_mounted_repo(self): + """Test _update_date_ranges calls update_repository_date_range for each mounted repo""" + mock_client = MagicMock() + mock_client.indices.exists.return_value = True + + mock_settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + style="oneup", + last_suffix="000001", + ) + + repo1 = Repository(name="deepfreeze-000001", bucket="bucket1", is_mounted=True) + repo2 = Repository(name="deepfreeze-000002", bucket="bucket2", is_mounted=True) + + with patch("deepfreeze_core.actions.rotate.get_settings") as mock_get: + mock_get.return_value = mock_settings + + with patch( + "deepfreeze_core.actions.rotate.s3_client_factory" + ) as mock_factory: + mock_s3 = MagicMock() + mock_factory.return_value = mock_s3 + + with patch( + "deepfreeze_core.actions.rotate.get_matching_repos" + ) as mock_repos: + mock_repos.return_value = [repo1, repo2] + + with patch( + "deepfreeze_core.actions.rotate.update_repository_date_range" + ) as mock_update: + # First repo updated, second repo unchanged + mock_update.side_effect = [True, False] + + rotate = Rotate(client=mock_client, porcelain=True) + rotate._load_settings() + + updated = rotate._update_date_ranges() + + assert mock_update.call_count == 2 + mock_update.assert_any_call(mock_client, repo1) + mock_update.assert_any_call(mock_client, repo2) + assert updated == ["deepfreeze-000001"] + + class TestThawAction: """Tests for the Thaw action class""" From 5d4df686cf22504725d9a0c5d7c945ae291d73bf Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 28 Jan 2026 08:18:26 -0500 Subject: [PATCH 27/30] =?UTF-8?q?=F0=9F=A5=85=20fix:=20handle=20null=20agg?= =?UTF-8?q?regation=20results=20in=20get=5Ftimestamp=5Frange?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When querying indices with no documents or no @timestamp field, Elasticsearch returns {"value": null} without a value_as_string key. This was causing an uncaught KeyError that silently propagated up to update_repository_date_range, making it impossible to distinguish between "no indices found" and "indices found but no @timestamp data". - Add early return if all indices filtered out (empty index list) - Use .get() instead of dict access for value_as_string - Add explicit null check with debug logging explaining why - Prevents KeyError from masking the real cause of empty date ranges Co-Authored-By: Claude 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) --- .../deepfreeze_core/utilities.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index de0d8fd..4260708 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -199,6 +199,10 @@ def get_timestamp_range(client: Elasticsearch, indices: list) -> tuple: filtered = [index for index in indices if client.indices.exists(index=index)] logging.debug("after removing non-existent indices: %s", len(filtered)) + if not filtered: + logging.debug("No existing indices remain after filtering") + return None, None + try: response = client.search( index=",".join(filtered), body=query, allow_partial_search_results=True @@ -208,12 +212,18 @@ def get_timestamp_range(client: Elasticsearch, indices: list) -> tuple: logging.error("Error retrieving timestamp range: %s", e) return None, None - earliest = response["aggregations"]["earliest"]["value_as_string"] - latest = response["aggregations"]["latest"]["value_as_string"] + earliest_val = response["aggregations"]["earliest"].get("value_as_string") + latest_val = response["aggregations"]["latest"].get("value_as_string") + + if not earliest_val or not latest_val: + logging.debug( + "Aggregation returned null timestamps (indices may have no @timestamp data)" + ) + return None, None - logging.debug("Earliest: %s, Latest: %s", earliest, latest) + logging.debug("Earliest: %s, Latest: %s", earliest_val, latest_val) - return datetime.fromisoformat(earliest), datetime.fromisoformat(latest) + return datetime.fromisoformat(earliest_val), datetime.fromisoformat(latest_val) def ensure_settings_index( From b59f41110a620d213f89a14d9eaca05220e679c6 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 28 Jan 2026 09:46:55 -0500 Subject: [PATCH 28/30] =?UTF-8?q?=F0=9F=90=9B=20fix:=20strip=20fm-clone=20?= =?UTF-8?q?prefix=20when=20matching=20snapshot=20indices=20to=20mounted=20?= =?UTF-8?q?indices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ILM force-merge creates snapshots with fm-clone-- prefix (e.g., fm-clone-i6je-.ds-df-test-2026.01.28-000001) but when those snapshots are mounted as searchable snapshots, ES strips the prefix (restored-.ds-df-test-2026.01.28-000001). This mismatch prevented update_repository_date_range from finding the mounted indices, resulting in empty date ranges. Strip the fm-clone-xxxx- prefix from snapshot index names before attempting to match them to mounted indices. This allows the code to correctly find restored- indices and query their @timestamp ranges. Co-Authored-By: Claude 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) --- .../deepfreeze-core/deepfreeze_core/utilities.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index 4260708..a192118 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -1130,6 +1130,19 @@ def update_repository_date_range(client: Elasticsearch, repo: Repository) -> boo # Find which indices are actually mounted (try multiple naming patterns) mounted_indices = [] for idx in snapshot_indices: + # ILM force-merge creates snapshots with fm-clone-xxxx- prefix, + # but mounted indices use the original name. Strip that prefix. + # Pattern: fm-clone-- -> + if idx.startswith("fm-clone-"): + # Find the second hyphen (after random chars) and strip prefix + parts = idx.split("-", 3) # ['fm', 'clone', 'random', 'rest'] + if len(parts) >= 4: + original_idx = parts[3] # Everything after fm-clone-xxxx- + loggit.debug( + "Stripped fm-clone prefix: %s -> %s", idx, original_idx + ) + idx = original_idx + # Try original name if client.indices.exists(index=idx): mounted_indices.append(idx) From ac3379c170062047c95b8f98565418c6300fb012 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 28 Jan 2026 14:13:49 -0500 Subject: [PATCH 29/30] =?UTF-8?q?=F0=9F=94=A5=20refactor:=20remove=20Phase?= =?UTF-8?q?=20column=20from=20ILM=20policies=20status=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Phase column showed which ILM phase contained the searchable_snapshot action at the time each versioned policy was created. This was configuration archaeology that provided no actionable information for operating or monitoring the system. Remove the column and associated phase_name tracking, keeping only the useful operational data: policy name, repository, and usage counts. Co-Authored-By: Claude 🤖 Commit generated with [Claude Code](https://claude.com/claude-code) --- packages/deepfreeze-core/deepfreeze_core/actions/status.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/status.py b/packages/deepfreeze-core/deepfreeze_core/actions/status.py index 88b5b0a..4f36186 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/status.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/status.py @@ -247,7 +247,6 @@ def _get_ilm_policies(self) -> list: policies.append( { "name": policy_name, - "phase": phase_name, "repository": snapshot_repo, "indices_count": len(in_use_by.get("indices", [])), "data_streams_count": len( @@ -440,7 +439,6 @@ def _display_rich( if ilm_policies: table = Table(title="ILM Policies (referencing deepfreeze)") table.add_column("Policy Name", style="cyan") - table.add_column("Phase", style="yellow") table.add_column("Repository", style="green") table.add_column("Indices", style="white") table.add_column("Data Streams", style="white") @@ -449,7 +447,6 @@ def _display_rich( for policy in sorted(ilm_policies, key=lambda x: x["name"]): table.add_row( policy["name"], - policy["phase"], policy["repository"], str(policy["indices_count"]), str(policy["data_streams_count"]), From 8cf1f27381d2344093bfe4aa04c341aed7c07452 Mon Sep 17 00:00:00 2001 From: Bret Wortman Date: Wed, 28 Jan 2026 17:38:37 -0500 Subject: [PATCH 30/30] feat: add --time/-rt flag to status command Show full date+time in status tables instead of date-only. Usage: deepfreeze status --time (or -rt) - Added -rt/--time click option to CLI status command - Added show_time parameter to Status class - Added _format_date_value() and _format_created_value() helpers - Date ranges show full ISO datetime when flag is set - Created timestamp shows full datetime when flag is set - Added tests for both date-only and time display modes --- .../deepfreeze-cli/deepfreeze/cli/main.py | 10 +++ .../deepfreeze_core/actions/status.py | 23 ++++- tests/cli/test_actions.py | 88 +++++++++++++++++++ tests/cli/test_cli.py | 1 + uv.lock | 8 ++ 5 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 uv.lock diff --git a/packages/deepfreeze-cli/deepfreeze/cli/main.py b/packages/deepfreeze-cli/deepfreeze/cli/main.py index 3431a24..638d59a 100644 --- a/packages/deepfreeze-cli/deepfreeze/cli/main.py +++ b/packages/deepfreeze-cli/deepfreeze/cli/main.py @@ -478,6 +478,14 @@ def rotate( default=False, help="Output plain text without formatting (suitable for scripting)", ) +@click.option( + "-rt", + "--time", + "show_time", + is_flag=True, + default=False, + help="Show full date+time in tables", +) @click.pass_context def status( ctx, @@ -488,6 +496,7 @@ def status( ilm, show_config_flag, porcelain, + show_time, ): """ Show the status of deepfreeze @@ -509,6 +518,7 @@ def status( show_buckets=buckets, show_ilm=ilm, show_config=show_config_flag, + show_time=show_time, ) try: diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/status.py b/packages/deepfreeze-core/deepfreeze_core/actions/status.py index 4f36186..7adfe44 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/status.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/status.py @@ -52,6 +52,7 @@ def __init__( show_buckets: bool = False, show_ilm: bool = False, show_config: bool = False, + show_time: bool = False, limit: int = None, **kwargs, # Accept extra kwargs for compatibility with curator CLI ) -> None: @@ -64,6 +65,7 @@ def __init__( self.client = client self.porcelain = porcelain self.limit = limit + self.show_time = show_time # Section flags - if none specified, show all any_section_specified = ( @@ -81,6 +83,17 @@ def __init__( self.loggit.debug("Deepfreeze Status initialized") + def _format_date_value(self, value: str) -> str: + if self.show_time: + return value.replace("T", " ") + return value[:10] + + def _format_created_value(self, value: str) -> str: + display = value.replace("T", " ") + if self.show_time: + return display + return display[:16] + def _load_settings(self) -> None: """Load settings from the status index.""" self.loggit.debug("Loading settings") @@ -323,7 +336,10 @@ def _display_rich( for repo in sorted(display_repos, key=lambda x: x["name"]): date_range = "" if repo.get("start") and repo.get("end"): - date_range = f"{repo['start'][:10]} - {repo['end'][:10]}" + date_range = ( + f"{self._format_date_value(repo['start'])} - " + f"{self._format_date_value(repo['end'])}" + ) mounted_str = ( "[green]Yes[/green]" @@ -388,7 +404,8 @@ def _display_rich( date_range = "" if req.get("start_date") and req.get("end_date"): date_range = ( - f"{req['start_date'][:10]} - {req['end_date'][:10]}" + f"{self._format_date_value(req['start_date'])} - " + f"{self._format_date_value(req['end_date'])}" ) repos_str = ", ".join(req.get("repos", [])[:3]) @@ -397,7 +414,7 @@ def _display_rich( created = req.get("created_at", "N/A") if created and created != "N/A": - created = created[:16].replace("T", " ") + created = self._format_created_value(created) table.add_row( req.get("request_id", "N/A"), diff --git a/tests/cli/test_actions.py b/tests/cli/test_actions.py index ca01e15..1245973 100644 --- a/tests/cli/test_actions.py +++ b/tests/cli/test_actions.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch import pytest +from rich.console import Console from deepfreeze import ( Cleanup, MissingIndexError, @@ -726,6 +727,93 @@ def test_setup_with_canned_acl(self): class TestStatusActionAdditional: """Additional tests for Status action (Task Group 18)""" + def test_status_rich_output_date_only(self): + """Test Status rich output truncates dates when show_time is False""" + status = Status( + client=MagicMock(), + porcelain=False, + show_repos=True, + show_thawed=True, + show_buckets=False, + show_ilm=False, + show_config=False, + ) + status.console = Console(stderr=True, record=True, width=200) + + repos = [ + { + "name": "repo-1", + "bucket": "bucket-1", + "base_path": "path", + "start": "2024-01-01T00:00:00+00:00", + "end": "2024-01-31T23:59:59+00:00", + "is_mounted": True, + "thaw_state": "frozen", + "storage_tier": "Archive", + } + ] + thaw_requests = [ + { + "request_id": "req-1", + "status": "completed", + "start_date": "2024-01-01T00:00:00+00:00", + "end_date": "2024-01-31T23:59:59+00:00", + "repos": ["repo-1"], + "created_at": "2024-02-01T12:34:56+00:00", + } + ] + + status._display_rich(repos, thaw_requests, buckets=[], ilm_policies=[]) + + output = status.console.export_text() + assert "2024-01-01 - 2024-01-31" in output + assert "2024-02-01 12:34" in output + assert "2024-01-01 00:00:00+00:00" not in output + assert "2024-02-01 12:34:56+00:00" not in output + + def test_status_rich_output_with_time(self): + """Test Status rich output shows full datetime when show_time is True""" + status = Status( + client=MagicMock(), + porcelain=False, + show_repos=True, + show_thawed=True, + show_buckets=False, + show_ilm=False, + show_config=False, + show_time=True, + ) + status.console = Console(stderr=True, record=True, width=200) + + repos = [ + { + "name": "repo-1", + "bucket": "bucket-1", + "base_path": "path", + "start": "2024-01-01T00:00:00+00:00", + "end": "2024-01-31T23:59:59+00:00", + "is_mounted": True, + "thaw_state": "frozen", + "storage_tier": "Archive", + } + ] + thaw_requests = [ + { + "request_id": "req-1", + "status": "completed", + "start_date": "2024-01-01T00:00:00+00:00", + "end_date": "2024-01-31T23:59:59+00:00", + "repos": ["repo-1"], + "created_at": "2024-02-01T12:34:56+00:00", + } + ] + + status._display_rich(repos, thaw_requests, buckets=[], ilm_policies=[]) + + output = status.console.export_text() + assert "2024-01-01 00:00:00+00:00 - 2024-01-31 23:59:59+00:00" in output + assert "2024-02-01 12:34:56+00:00" in output + def test_status_porcelain_mode(self): """Test Status with porcelain=True produces machine-readable output""" mock_client = MagicMock() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 59ee38b..3d8d839 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -197,6 +197,7 @@ def test_status_command_options(self, runner): assert "--limit" in result.output assert "--repos" in result.output assert "--porcelain" in result.output + assert "--time" in result.output def test_rotate_command_options(self, runner): """Test that rotate command has expected options.""" diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..f522afa --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.8" + +[[package]] +name = "deepfreeze-workspace" +version = "1.0.0" +source = { editable = "." }