diff --git a/.gitignore b/.gitignore index 7587e84..ab48a64 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ Thumbs.db *.local .env.local deepfreeze-blog-post.md +examples/config.yml diff --git a/README.md b/README.md index a74566e..3c23df1 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,23 @@ pip install git+https://github.com/elastic/deepfreeze.git#subdirectory=packages/ [View deepfreeze-cli documentation](packages/deepfreeze-cli/README.md) +## Supported Cloud Providers + +Deepfreeze supports multiple cloud storage providers: + +| Provider | Storage Type | Archive Tier | +|----------|--------------|--------------| +| **AWS** | S3 | Glacier, Glacier Deep Archive | +| **Azure** | Blob Storage | Archive tier | +| **GCP** | Cloud Storage | Archive storage class | + ## Features -- **Setup**: Configure ILM policies, index templates, and S3 buckets for deepfreeze +- **Setup**: Configure ILM policies, index templates, and storage buckets for deepfreeze - **Rotate**: Create new snapshot repositories on a schedule (weekly/monthly/yearly) - **Status**: View the current state of all deepfreeze components -- **Thaw**: Restore data from Glacier for analysis -- **Refreeze**: Return thawed data to Glacier storage +- **Thaw**: Restore data from archive storage for analysis +- **Refreeze**: Return thawed data to archive storage - **Cleanup**: Remove expired thaw requests and associated resources - **Repair Metadata**: Fix inconsistencies in the deepfreeze status index @@ -53,11 +63,22 @@ pip install git+https://github.com/elastic/deepfreeze.git#subdirectory=packages/ username: elastic password: changeme - deepfreeze: - provider: aws - bucket_name_prefix: my-deepfreeze - repo_name_prefix: deepfreeze - rotate_by: week + # Storage provider credentials (optional - can also use environment variables) + storage: + # AWS S3 + aws: + region: us-east-1 + # profile: my-profile # Or use access_key_id + secret_access_key + + # Azure Blob Storage + azure: + connection_string: "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=..." + # Or use account_name + account_key + + # Google Cloud Storage + gcp: + project: my-gcp-project + credentials_file: /path/to/service-account.json ``` 3. Initialize deepfreeze: diff --git a/packages/deepfreeze-cli/README.md b/packages/deepfreeze-cli/README.md index df35b38..85dd2ae 100644 --- a/packages/deepfreeze-cli/README.md +++ b/packages/deepfreeze-cli/README.md @@ -1,14 +1,22 @@ # Deepfreeze -Standalone Elasticsearch S3 Glacier archival and lifecycle management tool. +Standalone Elasticsearch cloud storage archival and lifecycle management tool. ## Overview -Deepfreeze provides cost-effective S3 Glacier archival and lifecycle management for Elasticsearch snapshot repositories without requiring full Curator installation. It is a lightweight, focused tool for managing long-term data retention in AWS S3 Glacier storage. +Deepfreeze provides cost-effective cloud storage archival and lifecycle management for Elasticsearch snapshot repositories without requiring full Curator installation. It is a lightweight, focused tool for managing long-term data retention in cloud archive storage. + +## Supported Cloud Providers + +| Provider | Storage Type | Archive Tier | +|----------|--------------|--------------| +| **AWS** | S3 | Glacier, Glacier Deep Archive | +| **Azure** | Blob Storage | Archive tier | +| **GCP** | Cloud Storage | Archive storage class | ## Features -- S3 Glacier archival for Elasticsearch snapshot repositories +- Cloud archive storage for Elasticsearch snapshot repositories - Repository rotation with configurable retention - Thaw frozen repositories for data retrieval - Automatic refreeze after data access @@ -52,14 +60,20 @@ deepfreeze --help - Python 3.8 or higher - Elasticsearch 8.x cluster -- AWS credentials configured (for S3 access) +- Cloud provider credentials (one of): + - **AWS**: AWS credentials via environment, config file, or IAM role + - **Azure**: Connection string or account name + key + - **GCP**: Application Default Credentials or service account JSON - Required Python packages (installed automatically): - elasticsearch8 - - boto3 + - boto3 (for AWS) - click - rich - voluptuous - pyyaml +- Optional packages for additional providers: + - azure-storage-blob (for Azure): `pip install deepfreeze-cli[azure]` + - google-cloud-storage (for GCP): `pip install deepfreeze-cli[gcp]` ## Configuration @@ -88,7 +102,7 @@ vim ~/.deepfreeze/config.yml ### Configuration Format -Create a YAML configuration file to specify Elasticsearch connection settings: +Create a YAML configuration file to specify Elasticsearch connection and storage provider settings: ```yaml elasticsearch: @@ -103,6 +117,29 @@ elasticsearch: # Or use Elastic Cloud # cloud_id: deployment:base64string +# Storage provider credentials (optional - can also use environment variables) +storage: + # AWS S3 configuration + aws: + region: us-east-1 + # profile: my-profile # Use named profile from ~/.aws/credentials + # Or explicit credentials: + # access_key_id: AKIA... + # secret_access_key: ... + + # Azure Blob Storage configuration + azure: + connection_string: "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=..." + # Or use account name + key: + # account_name: mystorageaccount + # account_key: ... + + # Google Cloud Storage configuration + gcp: + project: my-gcp-project + credentials_file: /path/to/service-account.json + location: US + logging: loglevel: INFO # logfile: /var/log/deepfreeze.log @@ -112,13 +149,30 @@ logging: Configuration can also be provided via environment variables: +**Elasticsearch:** - `DEEPFREEZE_ES_HOSTS` - Elasticsearch hosts (comma-separated) - `DEEPFREEZE_ES_USERNAME` - Elasticsearch username - `DEEPFREEZE_ES_PASSWORD` - Elasticsearch password - `DEEPFREEZE_ES_API_KEY` - Elasticsearch API key - `DEEPFREEZE_ES_CLOUD_ID` - Elastic Cloud ID -Environment variables override file configuration. +**AWS S3:** +- `AWS_ACCESS_KEY_ID` - AWS access key +- `AWS_SECRET_ACCESS_KEY` - AWS secret key +- `AWS_DEFAULT_REGION` - AWS region +- `AWS_PROFILE` - AWS profile name + +**Azure Blob Storage:** +- `AZURE_STORAGE_CONNECTION_STRING` - Full connection string +- `AZURE_STORAGE_ACCOUNT` - Storage account name (with AZURE_STORAGE_KEY) +- `AZURE_STORAGE_KEY` - Storage account key + +**Google Cloud Storage:** +- `GOOGLE_APPLICATION_CREDENTIALS` - Path to service account JSON +- `GOOGLE_CLOUD_PROJECT` - GCP project ID +- `GOOGLE_CLOUD_LOCATION` - Default bucket location + +Environment variables are used as fallback when config file credentials are not provided. ## Usage @@ -127,11 +181,26 @@ Environment variables override file configuration. Set up deepfreeze with ILM policy and index template configuration: ```bash +# AWS (default) deepfreeze --config config.yaml setup \ --ilm_policy_name my-ilm-policy \ --index_template_name my-template \ --bucket_name_prefix my-deepfreeze \ --repo_name_prefix my-deepfreeze + +# Azure +deepfreeze --config config.yaml setup \ + --provider azure \ + --ilm_policy_name my-ilm-policy \ + --bucket_name_prefix my-deepfreeze \ + --repo_name_prefix my-deepfreeze + +# GCP +deepfreeze --config config.yaml setup \ + --provider gcp \ + --ilm_policy_name my-ilm-policy \ + --bucket_name_prefix my-deepfreeze \ + --repo_name_prefix my-deepfreeze ``` ### Check Status @@ -216,11 +285,11 @@ deepfreeze --config config.yaml --dry-run repair-metadata | Command | Description | |---------|-------------| -| `setup` | Initialize deepfreeze environment | +| `setup` | Initialize deepfreeze environment (supports `--provider` option) | | `status` | Show current status of repositories and requests | | `rotate` | Create new repository and archive old ones | -| `thaw` | Initiate or check Glacier restore operations | -| `refreeze` | Return thawed repositories to Glacier | +| `thaw` | Initiate or check archive restore operations | +| `refreeze` | Return thawed repositories to archive storage | | `cleanup` | Remove expired repositories and old requests | | `repair-metadata` | Scan and repair metadata discrepancies | diff --git a/packages/deepfreeze-cli/deepfreeze/cli/main.py b/packages/deepfreeze-cli/deepfreeze/cli/main.py index 2cd8133..638d59a 100644 --- a/packages/deepfreeze-cli/deepfreeze/cli/main.py +++ b/packages/deepfreeze-cli/deepfreeze/cli/main.py @@ -203,15 +203,9 @@ def cli(ctx, config_path, dry_run): @click.option( "-o", "--provider", - type=click.Choice( - [ - "aws", - # "gcp", - # "azure", - ] - ), + type=click.Choice(["aws", "azure", "gcp"]), default="aws", - help="What provider to use (AWS only for now)", + help="Cloud storage provider to use (aws, azure, or gcp)", ) @click.option( "-t", @@ -306,6 +300,42 @@ def setup( client = get_client_from_context(ctx) + # Azure container names don't allow underscores - offer to convert them + if provider == "azure": + names_to_check = { + "bucket_name_prefix": bucket_name_prefix, + "repo_name_prefix": repo_name_prefix, + "base_path_prefix": base_path_prefix, + } + names_with_underscores = { + name: value + for name, value in names_to_check.items() + if value and "_" in value + } + if names_with_underscores: + converted = { + name: value.replace("_", "-") + for name, value in names_with_underscores.items() + } + click.echo( + "Azure container names cannot contain underscores. " + "The following names would be converted:" + ) + for name, value in names_with_underscores.items(): + click.echo(f" {name}: {value} -> {converted[name]}") + + if not click.confirm("Do you want to proceed with these converted names?"): + click.echo("Aborted. Please provide names without underscores.") + ctx.exit(1) + + # Apply conversions + if "bucket_name_prefix" in converted: + bucket_name_prefix = converted["bucket_name_prefix"] + if "repo_name_prefix" in converted: + repo_name_prefix = converted["repo_name_prefix"] + if "base_path_prefix" in converted: + base_path_prefix = converted["base_path_prefix"] + action = Setup( client=client, year=year, @@ -448,6 +478,14 @@ def rotate( default=False, help="Output plain text without formatting (suitable for scripting)", ) +@click.option( + "-rt", + "--time", + "show_time", + is_flag=True, + default=False, + help="Show full date+time in tables", +) @click.pass_context def status( ctx, @@ -458,6 +496,7 @@ def status( ilm, show_config_flag, porcelain, + show_time, ): """ Show the status of deepfreeze @@ -479,6 +518,7 @@ def status( show_buckets=buckets, show_ilm=ilm, show_config=show_config_flag, + show_time=show_time, ) try: diff --git a/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py b/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py index 0179fae..5f0e8ad 100644 --- a/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py +++ b/packages/deepfreeze-cli/deepfreeze/defaults/__init__.py @@ -125,7 +125,7 @@ def provider(): """ Cloud provider for deepfreeze. """ - return {Optional("provider", default="aws"): Any("aws")} + return {Optional("provider", default="aws"): Any("aws", "azure", "gcp")} def rotate_by(): diff --git a/packages/deepfreeze-core/README.md b/packages/deepfreeze-core/README.md index fa171cc..e58a5bc 100644 --- a/packages/deepfreeze-core/README.md +++ b/packages/deepfreeze-core/README.md @@ -1,6 +1,6 @@ # Deepfreeze Core -Core library for Elasticsearch S3 Glacier archival operations. +Core library for Elasticsearch cloud storage archival operations. ## Overview @@ -8,10 +8,28 @@ This package provides the shared functionality for deepfreeze operations, used b - **deepfreeze-cli**: Standalone CLI tool - **elasticsearch-curator**: Full Curator with deepfreeze support +## Supported Cloud Providers + +| Provider | Storage Type | Archive Tier | Package | +|----------|--------------|--------------|---------| +| **AWS** | S3 | Glacier, Deep Archive | boto3 (included) | +| **Azure** | Blob Storage | Archive tier | azure-storage-blob (optional) | +| **GCP** | Cloud Storage | Archive class | google-cloud-storage (optional) | + ## Installation ```bash +# Base installation (AWS support only) pip install deepfreeze-core + +# With Azure support +pip install deepfreeze-core[azure] + +# With GCP support +pip install deepfreeze-core[gcp] + +# With all providers +pip install deepfreeze-core[azure,gcp] ``` ## Usage @@ -21,11 +39,28 @@ This package is typically used as a dependency by other packages. For direct usa ```python from deepfreeze_core import ( Setup, Status, Rotate, Thaw, Refreeze, Cleanup, RepairMetadata, - s3_client_factory, create_es_client + s3_client_factory, create_es_client, get_storage_credentials ) # Create ES client -client = create_es_client(hosts=["https://localhost:9200"], username="elastic", password="changeme") +client = create_es_client( + hosts=["https://localhost:9200"], + username="elastic", + password="changeme" +) + +# Create storage client (AWS) +s3 = s3_client_factory("aws", region="us-east-1") + +# Create storage client (Azure) +s3 = s3_client_factory("azure", connection_string="...") + +# Create storage client (GCP) +s3 = s3_client_factory("gcp", project="my-project") + +# Or load credentials from config file +creds = get_storage_credentials("/path/to/config.yaml", "azure") +s3 = s3_client_factory("azure", **creds) # Create and run an action status = Status(client=client) @@ -37,17 +72,57 @@ status.do_action() ### Actions - `Setup` - Initialize deepfreeze environment - `Status` - Display status of repositories and thaw requests -- `Rotate` - Create new repository, archive old ones to Glacier -- `Thaw` - Restore data from Glacier storage -- `Refreeze` - Return thawed data to Glacier +- `Rotate` - Create new repository, archive old ones +- `Thaw` - Restore data from archive storage +- `Refreeze` - Return thawed data to archive storage - `Cleanup` - Remove expired repositories and requests - `RepairMetadata` - Fix metadata discrepancies +### Storage Clients +- `s3_client_factory(provider, **kwargs)` - Create storage client for any provider +- `AwsS3Client` - AWS S3 implementation +- `AzureBlobClient` - Azure Blob Storage implementation +- `GcpStorageClient` - Google Cloud Storage implementation + ### Utilities -- `s3_client_factory` - Create S3 client for AWS - `create_es_client` - Create Elasticsearch client +- `get_storage_credentials` - Load storage credentials from config file +- `load_storage_config` - Load all storage configuration from config file - Repository and Settings management functions +## Configuration + +Storage credentials can be provided via: + +1. **Constructor arguments** (highest priority) +2. **Config file** (storage section) +3. **Environment variables** (fallback) + +### Config File Format + +```yaml +storage: + aws: + region: us-east-1 + profile: my-profile # or access_key_id + secret_access_key + + azure: + connection_string: "DefaultEndpointsProtocol=https;..." + # or account_name + account_key + + gcp: + project: my-project + credentials_file: /path/to/service-account.json +``` + +### Environment Variables + +**AWS:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`, `AWS_PROFILE` + +**Azure:** `AZURE_STORAGE_CONNECTION_STRING` or `AZURE_STORAGE_ACCOUNT` + `AZURE_STORAGE_KEY` + +**GCP:** `GOOGLE_APPLICATION_CREDENTIALS`, `GOOGLE_CLOUD_PROJECT` + ## Development ```bash diff --git a/packages/deepfreeze-core/deepfreeze_core/__init__.py b/packages/deepfreeze-core/deepfreeze_core/__init__.py index 38d63ac..bdb18e4 100644 --- a/packages/deepfreeze-core/deepfreeze_core/__init__.py +++ b/packages/deepfreeze-core/deepfreeze_core/__init__.py @@ -33,7 +33,9 @@ ESClientWrapper, create_es_client, create_es_client_from_config, + get_storage_credentials, load_config_from_yaml, + load_storage_config, validate_connection, ) @@ -62,6 +64,18 @@ s3_client_factory, ) +# Conditional Azure export (azure-storage-blob is optional) +try: + from deepfreeze_core.azure_client import AzureBlobClient +except ImportError: + AzureBlobClient = None # type: ignore[misc,assignment] + +# Conditional GCP export (google-cloud-storage is optional) +try: + from deepfreeze_core.gcp_client import GcpStorageClient +except ImportError: + GcpStorageClient = None # type: ignore[misc,assignment] + # Export commonly used utilities from deepfreeze_core.utilities import ( check_restore_status, @@ -112,13 +126,17 @@ "Settings", # S3 Client "AwsS3Client", + "AzureBlobClient", + "GcpStorageClient", "S3Client", "s3_client_factory", # ES Client "ESClientWrapper", "create_es_client", "create_es_client_from_config", + "get_storage_credentials", "load_config_from_yaml", + "load_storage_config", "validate_connection", # Actions "Cleanup", diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py index 3241d18..0e20f5c 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/repair_metadata.py @@ -22,6 +22,7 @@ from deepfreeze_core.utilities import ( get_all_repos, get_settings, + update_repository_date_range, ) @@ -199,15 +200,62 @@ def _scan_repositories(self) -> list: ) continue - # Compare recorded state vs actual state - if repo.thaw_state != actual["determined_state"]: + # Compare recorded state vs actual storage state + # Note: "active" and "thawed" both use instant-access storage, so they're + # indistinguishable by storage class alone. We only flag a discrepancy + # when the storage state contradicts the recorded state. + is_discrepancy = False + suggested_state = None + + if repo.thaw_state in [THAW_STATE_ACTIVE, THAW_STATE_THAWED]: + # These states should have instant-access storage + if actual["glacier"] == actual["total_objects"] and actual["total_objects"] > 0: + # All objects in archive - should be frozen + is_discrepancy = True + suggested_state = THAW_STATE_FROZEN + elif actual["restoring"] > 0: + # Some objects restoring - should be thawing + is_discrepancy = True + suggested_state = THAW_STATE_THAWING + # If instant_access, no discrepancy - active/thawed both valid + + elif repo.thaw_state == THAW_STATE_FROZEN: + # This state should have archive storage + if actual["instant_access"] == actual["total_objects"] and actual["total_objects"] > 0: + # All objects accessible - was restored (thawed) + is_discrepancy = True + suggested_state = THAW_STATE_THAWED + elif actual["restoring"] > 0: + # Some objects restoring - thaw in progress + is_discrepancy = True + suggested_state = THAW_STATE_THAWING + + elif repo.thaw_state == THAW_STATE_THAWING: + # This state should have some objects restoring or all accessible + if actual["glacier"] == actual["total_objects"] and actual["total_objects"] > 0: + # All still in archive - thaw failed or not started + is_discrepancy = True + suggested_state = THAW_STATE_FROZEN + elif actual["instant_access"] == actual["total_objects"] and actual["total_objects"] > 0: + # All accessible - thaw completed + is_discrepancy = True + suggested_state = THAW_STATE_THAWED + + elif repo.thaw_state == THAW_STATE_EXPIRED: + # Expired should have archive storage (restore expired) + if actual["instant_access"] == actual["total_objects"] and actual["total_objects"] > 0: + # Still accessible - not actually expired + is_discrepancy = True + suggested_state = THAW_STATE_THAWED + + if is_discrepancy and suggested_state: discrepancies.append( { "repo": repo.name, "bucket": repo.bucket, "base_path": repo.base_path, "recorded_state": repo.thaw_state, - "actual_state": actual["determined_state"], + "actual_state": suggested_state, "storage_classes": actual["storage_classes"], "total_objects": actual["total_objects"], "instant_access": actual["instant_access"], @@ -219,7 +267,7 @@ def _scan_repositories(self) -> list: "Discrepancy found for %s: recorded=%s, actual=%s", repo.name, repo.thaw_state, - actual["determined_state"], + suggested_state, ) return discrepancies @@ -290,6 +338,92 @@ def _repair_discrepancy(self, discrepancy: dict, dry_run: bool = False) -> dict: return result + def _update_date_ranges(self, dry_run: bool = False) -> list: + """ + Update date ranges for all mounted repositories. + + Date ranges can only be computed for mounted repositories since we need + to query @timestamp values from mounted indices. + + :param dry_run: If True, don't actually update anything + :return: List of result dictionaries + """ + results = [] + repos = get_all_repos(self.client) + + for repo in repos: + # Only process mounted repos (we can query their indices) + if not repo.is_mounted: + self.loggit.debug( + "Skipping date range update for unmounted repo %s", repo.name + ) + continue + + # Skip if dates already set + if repo.start and repo.end: + self.loggit.debug( + "Repo %s already has date range: %s to %s", + repo.name, + repo.start.isoformat(), + repo.end.isoformat(), + ) + continue + + self.loggit.info("Checking date range for repo %s", repo.name) + + result = { + "repo": repo.name, + "success": False, + "old_start": repo.start.isoformat() if repo.start else None, + "old_end": repo.end.isoformat() if repo.end else None, + "new_start": None, + "new_end": None, + "error": None, + } + + # Verify the repo actually exists in ES before trying to query it + try: + es_repos = self.client.snapshot.get_repository(name=repo.name) + if repo.name not in es_repos: + self.loggit.debug( + "Repo %s marked as mounted but not found in ES, skipping", + repo.name, + ) + continue + except Exception as e: + self.loggit.debug( + "Repo %s not accessible in ES: %s, skipping", repo.name, e + ) + continue + + if dry_run: + # In dry run, report that we would update + result["success"] = True + results.append(result) + continue + + try: + # Use the utility function to update date range + if update_repository_date_range(self.client, repo): + result["success"] = True + result["new_start"] = repo.start.isoformat() if repo.start else None + result["new_end"] = repo.end.isoformat() if repo.end else None + self.loggit.info( + "Updated date range for %s: %s to %s", + repo.name, + result["new_start"], + result["new_end"], + ) + else: + result["error"] = "No timestamp data found in indices" + except Exception as e: + result["error"] = str(e) + self.loggit.error("Failed to update date range for %s: %s", repo.name, e) + + results.append(result) + + return results + def do_dry_run(self) -> None: """ Scan and report discrepancies without making changes. @@ -309,6 +443,9 @@ def do_dry_run(self) -> None: discrepancies = self._scan_repositories() + # Also check for missing date ranges on mounted repos + date_range_updates = self._update_date_ranges(dry_run=True) + if self.porcelain: for d in discrepancies: if d.get("error"): @@ -318,14 +455,16 @@ def do_dry_run(self) -> None: f"DISCREPANCY\t{d['repo']}\t{d['recorded_state']}\t" f"{d['actual_state']}\t{d.get('total_objects', 0)} objects" ) - print(f"SUMMARY\t{len(discrepancies)} discrepancies found") + for u in date_range_updates: + print(f"DATE_RANGE\t{u['repo']}\tmissing") + print(f"SUMMARY\t{len(discrepancies)} discrepancies\t{len(date_range_updates)} missing date ranges") return - if not discrepancies: + if not discrepancies and not date_range_updates: self.console.print( Panel( - "[green]No discrepancies found![/green]\n\n" - "All repository states in the status index match the actual S3 storage classes.", + "[green]No issues found![/green]\n\n" + "All repository states and date ranges are correct.", title="[bold green]Scan Complete[/bold green]", border_style="green", expand=False, @@ -365,13 +504,33 @@ def do_dry_run(self) -> None: storage_str, ) - self.console.print(table) - self.console.print() + if discrepancies: + self.console.print(table) + self.console.print() + + # Display date range updates table if any + if date_range_updates: + date_table = Table(title="Missing Date Ranges") + date_table.add_column("Repository", style="cyan") + date_table.add_column("Status", style="yellow") + + for u in date_range_updates: + date_table.add_row(u["repo"], "Missing - will compute from indices") + + self.console.print(date_table) + self.console.print() + + # Summary + summary_parts = [] + if discrepancies: + summary_parts.append(f"{len(discrepancies)} state discrepancies") + if date_range_updates: + summary_parts.append(f"{len(date_range_updates)} missing date ranges") self.console.print( Panel( - f"[bold]Found {len(discrepancies)} discrepancies[/bold]\n\n" - f"Run without [yellow]--dry-run[/yellow] to repair these discrepancies.", + f"[bold]Found {' and '.join(summary_parts)}[/bold]\n\n" + f"Run without [yellow]--dry-run[/yellow] to repair.", title="[bold blue]Dry Run Summary[/bold blue]", border_style="blue", expand=False, @@ -404,14 +563,17 @@ def do_action(self) -> None: discrepancies = self._scan_repositories() - if not discrepancies: + # Update date ranges for mounted repos (actual update, not dry run) + date_range_results = self._update_date_ranges(dry_run=False) + + if not discrepancies and not date_range_results: if self.porcelain: - print("COMPLETE\t0 discrepancies") + print("COMPLETE\t0 discrepancies\t0 date ranges") else: self.console.print( Panel( - "[green]No discrepancies found![/green]\n\n" - "All repository states in the status index match the actual S3 storage classes.", + "[green]No issues found![/green]\n\n" + "All repository states and date ranges are correct.", title="[bold green]Scan Complete[/bold green]", border_style="green", expand=False, @@ -419,68 +581,103 @@ def do_action(self) -> None: ) return - if not self.porcelain: - self.console.print( - f"[bold]Found {len(discrepancies)} discrepancies. Repairing...[/bold]" - ) - - # Repair each discrepancy - results = [] - for d in discrepancies: - result = self._repair_discrepancy(d) - results.append(result) + # Repair state discrepancies + state_results = [] + if discrepancies: + if not self.porcelain: + self.console.print( + f"[bold]Found {len(discrepancies)} state discrepancies. Repairing...[/bold]" + ) + for d in discrepancies: + result = self._repair_discrepancy(d) + state_results.append(result) # Display results + state_success = sum(1 for r in state_results if r["success"]) + state_fail = sum(1 for r in state_results if not r["success"]) + date_success = sum(1 for r in date_range_results if r["success"]) + date_fail = sum(1 for r in date_range_results if not r["success"]) + if self.porcelain: - for r in results: + for r in state_results: status = "SUCCESS" if r["success"] else "FAILED" - print(f"{status}\t{r['repo']}\t{r['old_state']}\t{r['new_state']}") - success_count = sum(1 for r in results if r["success"]) - fail_count = sum(1 for r in results if not r["success"]) - print(f"COMPLETE\t{success_count} repaired\t{fail_count} failed") + print(f"STATE\t{status}\t{r['repo']}\t{r['old_state']}\t{r['new_state']}") + for r in date_range_results: + status = "SUCCESS" if r["success"] else "FAILED" + error_msg = f"\t{r['error']}" if r.get("error") else "" + print(f"DATE_RANGE\t{status}\t{r['repo']}\t{r.get('new_start')}\t{r.get('new_end')}{error_msg}") + print(f"COMPLETE\t{state_success + date_success} repaired\t{state_fail + date_fail} failed") else: - # Summary - success_count = sum(1 for r in results if r["success"]) - fail_count = sum(1 for r in results if not r["success"]) - - if fail_count == 0: + total_fail = state_fail + date_fail + + if total_fail == 0: + summary_parts = [] + if state_success: + summary_parts.append(f"{state_success} state discrepancies") + if date_success: + summary_parts.append(f"{date_success} date ranges") self.console.print( Panel( - f"[bold green]Successfully repaired {success_count} discrepancies![/bold green]", + f"[bold green]Successfully repaired {' and '.join(summary_parts)}![/bold green]", title="[bold green]Repair Complete[/bold green]", border_style="green", expand=False, ) ) else: - # Show detailed results - table = Table(title="Repair Results") - table.add_column("Repository", style="cyan") - table.add_column("Old State", style="yellow") - table.add_column("New State", style="green") - table.add_column("Status", style="white") - - for r in results: - if r["success"]: - status = "[green]Repaired[/green]" - else: - status = f"[red]Failed: {r.get('error', 'Unknown')}[/red]" - - table.add_row( - r["repo"], - r.get("old_state", "N/A"), - r.get("new_state", "N/A"), - status, - ) - - self.console.print(table) - self.console.print() + # Show detailed results for state discrepancies + if state_results: + table = Table(title="State Repair Results") + table.add_column("Repository", style="cyan") + table.add_column("Old State", style="yellow") + table.add_column("New State", style="green") + table.add_column("Status", style="white") + + for r in state_results: + if r["success"]: + status = "[green]Repaired[/green]" + else: + status = f"[red]Failed: {r.get('error', 'Unknown')}[/red]" + + table.add_row( + r["repo"], + r.get("old_state", "N/A"), + r.get("new_state", "N/A"), + status, + ) + + self.console.print(table) + self.console.print() + + # Show detailed results for date range updates + if date_range_results: + date_table = Table(title="Date Range Repair Results") + date_table.add_column("Repository", style="cyan") + date_table.add_column("Start", style="green") + date_table.add_column("End", style="green") + date_table.add_column("Status", style="white") + + for r in date_range_results: + if r["success"]: + status = "[green]Updated[/green]" + else: + status = f"[red]Failed: {r.get('error', 'Unknown')}[/red]" + + date_table.add_row( + r["repo"], + r.get("new_start") or "N/A", + r.get("new_end") or "N/A", + status, + ) + + self.console.print(date_table) + self.console.print() self.console.print( Panel( f"[bold]Repair completed with some failures[/bold]\n\n" - f"Repaired: {success_count}\n" - f"Failed: {fail_count}\n\n" + f"State repairs: {state_success} succeeded, {state_fail} failed\n" + f"Date ranges: {date_success} updated, {date_fail} failed\n\n" f"Check logs for details on failures.", title="[bold yellow]Repair Complete[/bold yellow]", border_style="yellow", @@ -489,9 +686,10 @@ def do_action(self) -> None: ) self.loggit.info( - "RepairMetadata complete: %d repaired, %d failed", - success_count, - fail_count, + "RepairMetadata complete: %d state repairs, %d date ranges updated, %d failed", + state_success, + date_success, + state_fail + date_fail, ) except (MissingIndexError, MissingSettingsError) as e: diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py index 5017abc..0b7b80d 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/rotate.py @@ -22,12 +22,16 @@ get_composable_templates, get_ilm_policy, get_index_templates, + get_matching_repo_names, get_matching_repos, get_next_suffix, get_settings, + is_policy_safe_to_delete, push_to_glacier, + repo_has_active_indices, save_settings, unmount_repo, + update_repository_date_range, update_template_ilm_policy, ) @@ -120,21 +124,25 @@ def _create_new_repository(self, dry_run: bool = False) -> tuple: new_bucket_name = self.settings.bucket_name_prefix base_path = f"{self.settings.base_path_prefix}-{next_suffix}" + # Get provider-specific storage type for logging + storage_type = getattr(self.s3, "STORAGE_TYPE", "bucket").lower() + self.loggit.info( - "Creating new repository %s (bucket: %s, base_path: %s)", + "Creating new repository %s (%s: %s, base_path: %s)", new_repo_name, + storage_type, new_bucket_name, base_path, ) if not dry_run: - # Create bucket if rotating by bucket + # Create storage container/bucket if rotating by bucket if self.settings.rotate_by == "bucket": if not self.s3.bucket_exists(new_bucket_name): self.s3.create_bucket(new_bucket_name) - self.loggit.info("Created S3 bucket %s", new_bucket_name) + self.loggit.info("Created %s %s", storage_type, new_bucket_name) else: - self.loggit.info("S3 bucket %s already exists", new_bucket_name) + self.loggit.info("%s %s already exists", storage_type.capitalize(), new_bucket_name) # Create repository in Elasticsearch create_repo( @@ -144,6 +152,7 @@ def _create_new_repository(self, dry_run: bool = False) -> tuple: base_path, self.settings.canned_acl, self.settings.storage_class, + provider=self.settings.provider, ) # Update last_suffix in settings @@ -331,12 +340,27 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: # Keep the newest 'keep' repos mounted repos_to_archive = repos[: -self.keep] if len(repos) > self.keep else [] + skipped_repos = [] + for repo in repos_to_archive: + # Check if repo has active indices BEFORE archiving + # (can't unmount a repo with active searchable snapshot indices) + has_active, active_indices = repo_has_active_indices(self.client, repo.name) + if has_active: + self.loggit.warning( + "Skipping archive of %s - has %d active indices: %s", + repo.name, + len(active_indices), + active_indices[:3], + ) + skipped_repos.append(repo.name) + continue + self.loggit.info("Archiving repository %s to Glacier", repo.name) if not dry_run: try: - # Push all objects to Glacier + # Push all objects to archive tier push_to_glacier(self.s3, repo) # Unmount the repository @@ -357,8 +381,113 @@ def _archive_old_repos(self, dry_run: bool = False) -> list: else: archived_repos.append(repo.name) + if skipped_repos: + self.loggit.info( + "Skipped %d repos with active indices (will retry on next rotation): %s", + len(skipped_repos), + skipped_repos, + ) + return archived_repos + def _update_date_ranges(self) -> list: + """ + Update date ranges for all mounted repositories. + + Queries each mounted repo's indices to capture min/max @timestamp values. + This must run while repos still have searchable snapshot indices, i.e. + before the archive step unmounts them. + + :return: List of repo names whose date ranges were updated + """ + updated_repos = [] + + repos = get_matching_repos( + self.client, self.settings.repo_name_prefix, mounted=True + ) + + for repo in repos: + try: + if update_repository_date_range(self.client, repo): + self.loggit.info("Updated date range for %s", repo.name) + updated_repos.append(repo.name) + except Exception as e: + self.loggit.error( + "Failed to update date range for %s: %s", repo.name, e + ) + + return updated_repos + + def _cleanup_orphaned_policies(self, dry_run: bool = False) -> list: + """ + Delete ILM policies that reference unmounted deepfreeze repositories. + + This is called after archiving repos to clean up the old versioned + policies that now reference non-existent repositories. + + :param dry_run: If True, don't actually delete anything + :return: List of deleted policy names + """ + deleted_policies = [] + + if not self.settings.ilm_policy_name: + return deleted_policies + + try: + all_policies = self.client.ilm.get_lifecycle() + + # Get currently mounted repos + mounted_repos = set(get_matching_repo_names( + self.client, self.settings.repo_name_prefix + )) + + for policy_name, policy_data in all_policies.items(): + # Only check versioned policies matching our prefix + if not policy_name.startswith(f"{self.settings.ilm_policy_name}-"): + continue + + # Check if policy references a deepfreeze repo + policy_body = policy_data.get("policy", {}) + phases = policy_body.get("phases", {}) + + for _phase_name, phase_config in phases.items(): + actions = phase_config.get("actions", {}) + if "searchable_snapshot" in actions: + snapshot_repo = actions["searchable_snapshot"].get( + "snapshot_repository" + ) + # Check if it references a repo that's no longer mounted + if ( + snapshot_repo + and snapshot_repo.startswith(self.settings.repo_name_prefix) + and snapshot_repo not in mounted_repos + ): + # Check if policy is safe to delete + if is_policy_safe_to_delete(self.client, policy_name): + self.loggit.info( + "Deleting orphaned ILM policy %s (references unmounted repo %s)", + policy_name, + snapshot_repo, + ) + if not dry_run: + try: + self.client.ilm.delete_lifecycle(name=policy_name) + deleted_policies.append(policy_name) + except Exception as e: + self.loggit.error( + "Failed to delete policy %s: %s", + policy_name, + e, + ) + else: + deleted_policies.append(policy_name) + break + + except Exception as e: + self.loggit.warning("Error cleaning up orphaned policies: %s", e) + + return deleted_policies + def do_dry_run(self) -> None: """ Perform a dry-run of the rotation. @@ -429,6 +558,25 @@ def do_dry_run(self) -> None: ) ) + # Show orphaned policies that would be deleted + orphaned_policies = self._cleanup_orphaned_policies(dry_run=True) + if orphaned_policies: + if self.porcelain: + for policy in orphaned_policies: + print(f"DRY_RUN\tdelete_policy\t{policy}") + else: + policy_list = "\n".join( + [f" - [red]{p}[/red]" for p in orphaned_policies] + ) + self.console.print( + Panel( + f"[bold]Would delete {len(orphaned_policies)} orphaned ILM policies:[/bold]\n{policy_list}", + title="[bold blue]Dry Run - Delete Orphaned Policies[/bold blue]", + border_style="blue", + expand=False, + ) + ) + except (MissingIndexError, MissingSettingsError) as e: if self.porcelain: print(f"ERROR\t{type(e).__name__}\t{str(e)}") @@ -488,6 +636,26 @@ def do_action(self) -> None: ) ) + # Update date ranges for all mounted repos (before archiving + # removes searchable snapshot indices) + updated_date_ranges = self._update_date_ranges() + if updated_date_ranges: + if self.porcelain: + for repo in updated_date_ranges: + print(f"UPDATED\tdate_range\t{repo}") + else: + range_list = "\n".join( + [f" - [cyan]{r}[/cyan]" for r in updated_date_ranges] + ) + self.console.print( + Panel( + f"[bold]Updated date ranges for {len(updated_date_ranges)} repositories:[/bold]\n{range_list}", + title="[bold green]Date Ranges Updated[/bold green]", + border_style="green", + expand=False, + ) + ) + # Archive old repositories archived = self._archive_old_repos() if archived: @@ -507,6 +675,25 @@ def do_action(self) -> None: ) ) + # Clean up orphaned ILM policies (policies referencing unmounted repos) + deleted_policies = self._cleanup_orphaned_policies() + if deleted_policies: + if self.porcelain: + for policy in deleted_policies: + print(f"DELETED\tilm_policy\t{policy}") + else: + policy_list = "\n".join( + [f" - [red]{p}[/red]" for p in deleted_policies] + ) + self.console.print( + Panel( + f"[bold]Deleted {len(deleted_policies)} orphaned ILM policies:[/bold]\n{policy_list}", + title="[bold green]Orphaned Policies Cleaned Up[/bold green]", + border_style="green", + expand=False, + ) + ) + # Final summary if not self.porcelain: self.console.print( @@ -514,7 +701,9 @@ def do_action(self) -> None: f"[bold green]Rotation completed successfully![/bold green]\n\n" f"New repository: [cyan]{new_repo}[/cyan]\n" f"Policies updated: {len(updated_policies)}\n" - f"Repositories archived: {len(archived)}\n\n" + f"Date ranges updated: {len(updated_date_ranges)}\n" + f"Repositories archived: {len(archived)}\n" + f"Orphaned policies deleted: {len(deleted_policies)}\n\n" f"[bold]Next steps:[/bold]\n" f" - Verify ILM policies are using the new repository\n" f" - Monitor searchable snapshot transitions\n" diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py index a8d2e25..31b9349 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/setup.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/setup.py @@ -143,10 +143,10 @@ def _check_preconditions(self) -> None: errors.append( { "issue": f"Status index [cyan]{STATUS_INDEX}[/cyan] already exists", - "solution": f"Delete the existing index before running setup:\n" - f" [yellow]deepfreeze --host DELETE index --name {STATUS_INDEX}[/yellow]\n" - f" or use the Elasticsearch API:\n" - f" [yellow]curl -X DELETE 'http://:9200/{STATUS_INDEX}'[/yellow]", + "solution": f"Delete the existing index using the Elasticsearch API:\n" + f" [yellow]curl -X DELETE 'https://:9200/{STATUS_INDEX}'[/yellow]\n\n" + "Or via Kibana Dev Tools:\n" + f" [yellow]DELETE /{STATUS_INDEX}[/yellow]", } ) @@ -179,11 +179,13 @@ def _check_preconditions(self) -> None: # Third, check if the bucket already exists self.loggit.debug("Checking if bucket %s exists", self.new_bucket_name) if self.s3.bucket_exists(self.new_bucket_name): + storage_type = self.s3.STORAGE_TYPE + delete_cmd = self.s3.STORAGE_DELETE_CMD.format(bucket=self.new_bucket_name) errors.append( { - "issue": f"S3 bucket [cyan]{self.new_bucket_name}[/cyan] already exists", - "solution": f"Delete the existing bucket before running setup:\n" - f" [yellow]aws s3 rb s3://{self.new_bucket_name} --force[/yellow]\n" + "issue": f"{storage_type} [cyan]{self.new_bucket_name}[/cyan] already exists", + "solution": f"Delete the existing {storage_type.lower()} before running setup:\n" + f" [yellow]{delete_cmd}[/yellow]\n" "\n[bold]WARNING:[/bold] This will delete all data in the bucket!\n" "Or use a different bucket_name_prefix in your configuration.", } @@ -249,9 +251,14 @@ def _check_preconditions(self) -> None: template_type, ) - # Fifth, check for S3 repository plugin (only for ES 7.x and below) - # NOTE: Elasticsearch 8.x+ has built-in S3 repository support, no plugin needed - self.loggit.debug("Checking S3 repository support") + # Fifth, check for repository plugin based on provider + # NOTE: Elasticsearch 8.x+ has built-in repository support for all providers + # Get plugin info from the storage client class + plugin_name = self.s3.ES_PLUGIN_NAME + plugin_display = self.s3.ES_PLUGIN_DISPLAY_NAME + doc_url = self.s3.ES_PLUGIN_DOC_URL + + self.loggit.debug("Checking %s repository support", plugin_display) try: # Get Elasticsearch version cluster_info = self.client.info() @@ -259,47 +266,55 @@ def _check_preconditions(self) -> None: major_version = int(es_version.split(".")[0]) if major_version < 8: - # ES 7.x and below require the repository-s3 plugin + # ES 7.x and below require repository plugins self.loggit.debug( - "Elasticsearch %s detected - checking for S3 repository plugin", + "Elasticsearch %s detected - checking for %s repository plugin", es_version, + plugin_display, ) # Get cluster plugins nodes_info = self.client.nodes.info(node_id="_all", metric="plugins") - # Check if any node has the S3 plugin - has_s3_plugin = False + # Check if any node has the required plugin + has_plugin = False for node_id, node_data in nodes_info.get("nodes", {}).items(): plugins = node_data.get("plugins", []) for plugin in plugins: - if plugin.get("name") == "repository-s3": - has_s3_plugin = True - self.loggit.debug("Found S3 plugin on node %s", node_id) + if plugin.get("name") == plugin_name: + has_plugin = True + self.loggit.debug( + "Found %s plugin on node %s", plugin_name, node_id + ) break - if has_s3_plugin: + if has_plugin: break - if not has_s3_plugin: + if not has_plugin: errors.append( { - "issue": "Elasticsearch S3 repository plugin is not installed", - "solution": "Install the S3 repository plugin on all Elasticsearch nodes:\n" - " [yellow]bin/elasticsearch-plugin install repository-s3[/yellow]\n" + "issue": f"Elasticsearch {plugin_display} repository plugin is not installed", + "solution": f"Install the {plugin_display} repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install {plugin_name}[/yellow]\n" " Then restart all Elasticsearch nodes.\n" - " See: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html", + f" See: {doc_url}", } ) else: - self.loggit.debug("S3 repository plugin is installed") + self.loggit.debug( + "%s repository plugin is installed", plugin_display + ) else: - # ES 8.x+ has built-in S3 support + # ES 8.x+ has built-in repository support self.loggit.debug( - "Elasticsearch %s detected - S3 repository support is built-in", + "Elasticsearch %s detected - %s repository support is built-in", es_version, + plugin_display, ) except Exception as e: - self.loggit.warning("Could not verify S3 repository support: %s", e) + self.loggit.warning( + "Could not verify %s repository support: %s", plugin_display, e + ) # Don't add to errors - this is a soft check that may fail due to permissions # If any errors were found, display them all and raise exception @@ -371,6 +386,7 @@ def do_dry_run(self) -> None: self.base_path, self.settings.canned_acl, self.settings.storage_class, + provider=self.settings.provider, dry_run=True, ) @@ -433,19 +449,18 @@ def do_action(self) -> None: ) except Exception as e: if self.porcelain: - print(f"ERROR\ts3_bucket\t{self.new_bucket_name}\t{str(e)}") + print(f"ERROR\tstorage\t{self.new_bucket_name}\t{str(e)}") else: + # Get provider-specific error info from the storage client + storage_type = self.s3.STORAGE_TYPE + solutions = self.s3.STORAGE_CREATION_HELP + self.console.print( Panel( - f"[bold]Failed to create S3 bucket [cyan]{self.new_bucket_name}[/cyan][/bold]\n\n" + f"[bold]Failed to create {storage_type} [cyan]{self.new_bucket_name}[/cyan][/bold]\n\n" f"Error: {escape(str(e))}\n\n" - f"[bold]Possible Solutions:[/bold]\n" - f" - Check AWS credentials and permissions\n" - f" - Verify IAM policy allows s3:CreateBucket\n" - f" - Check if bucket name is globally unique\n" - f" - Verify AWS region settings\n" - f" - Check AWS account limits for S3 buckets", - title="[bold red]S3 Bucket Creation Error[/bold red]", + f"[bold]{solutions}[/bold]", + title=f"[bold red]{storage_type.title()} Creation Error[/bold red]", border_style="red", expand=False, ) @@ -471,6 +486,7 @@ def do_action(self) -> None: self.base_path, self.settings.canned_acl, self.settings.storage_class, + provider=self.settings.provider, ) self.loggit.info( "Successfully created repository %s", self.new_repo_name @@ -479,16 +495,28 @@ def do_action(self) -> None: if self.porcelain: print(f"ERROR\trepository\t{self.new_repo_name}\t{str(e)}") else: + # Get provider-specific error info from the storage client + plugin_name = self.s3.ES_PLUGIN_NAME + plugin_display = self.s3.ES_PLUGIN_DISPLAY_NAME + doc_url = self.s3.ES_PLUGIN_DOC_URL + storage_type = self.s3.STORAGE_TYPE + keystore_instructions = self.s3.ES_KEYSTORE_INSTRUCTIONS + + solutions = ( + f"[bold]Possible Solutions:[/bold]\n" + f" 1. Install the {plugin_display} repository plugin on all Elasticsearch nodes:\n" + f" [yellow]bin/elasticsearch-plugin install {plugin_name}[/yellow]\n\n" + f" 2. {keystore_instructions}\n\n" + f" 3. Verify {storage_type} [cyan]{self.new_bucket_name}[/cyan] is accessible\n\n" + f"[bold]Documentation:[/bold]\n" + f" {doc_url}" + ) + self.console.print( Panel( f"[bold]Failed to create repository [cyan]{self.new_repo_name}[/cyan][/bold]\n\n" f"Error: {escape(str(e))}\n\n" - f"[bold]Possible Solutions:[/bold]\n" - f" - Verify Elasticsearch has S3 plugin installed\n" - f" - Check AWS credentials are configured in Elasticsearch keystore\n" - f" - Verify S3 bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n" - f" - Check repository settings (ACL, storage class, etc.)\n" - f" - Review Elasticsearch logs for detailed error messages", + f"{solutions}", title="[bold red]Repository Creation Error[/bold red]", border_style="red", expand=False, @@ -715,13 +743,14 @@ def do_action(self) -> None: if self.porcelain: print(f"ERROR\tunexpected\t{str(e)}") else: + provider_name = self.s3.ES_PLUGIN_DISPLAY_NAME self.console.print( Panel( f"[bold]An unexpected error occurred during setup[/bold]\n\n" f"Error: {escape(str(e))}\n\n" f"[bold]What to do:[/bold]\n" f" - Check the logs for detailed error information\n" - f" - Verify all prerequisites are met (AWS credentials, ES connection, etc.)\n" + f" - Verify all prerequisites are met ({provider_name} credentials, ES connection, etc.)\n" f" - You may need to manually clean up any partially created resources\n" f" - Run [yellow]deepfreeze cleanup[/yellow] to remove any partial state", title="[bold red]Unexpected Setup Error[/bold red]", diff --git a/packages/deepfreeze-core/deepfreeze_core/actions/status.py b/packages/deepfreeze-core/deepfreeze_core/actions/status.py index edb38b2..7adfe44 100644 --- a/packages/deepfreeze-core/deepfreeze_core/actions/status.py +++ b/packages/deepfreeze-core/deepfreeze_core/actions/status.py @@ -52,6 +52,7 @@ def __init__( show_buckets: bool = False, show_ilm: bool = False, show_config: bool = False, + show_time: bool = False, limit: int = None, **kwargs, # Accept extra kwargs for compatibility with curator CLI ) -> None: @@ -64,6 +65,7 @@ def __init__( self.client = client self.porcelain = porcelain self.limit = limit + self.show_time = show_time # Section flags - if none specified, show all any_section_specified = ( @@ -81,6 +83,17 @@ def __init__( self.loggit.debug("Deepfreeze Status initialized") + def _format_date_value(self, value: str) -> str: + if self.show_time: + return value.replace("T", " ") + return value[:10] + + def _format_created_value(self, value: str) -> str: + display = value.replace("T", " ") + if self.show_time: + return display + return display[:16] + def _load_settings(self) -> None: """Load settings from the status index.""" self.loggit.debug("Loading settings") @@ -96,18 +109,78 @@ def _load_settings(self) -> None: # Initialize S3 client with provider from settings self.s3 = s3_client_factory(self.settings.provider) + def _get_repo_storage_tier(self, bucket: str, base_path: str) -> str: + """ + Sample blobs in a repository to determine the predominant storage tier. + + Returns a summary like "Archive", "Hot", "Cool", or "Mixed". + """ + try: + # Normalize path + path = base_path.strip("/") + if path: + path += "/" + + # List first few objects to sample storage tier + objects = self.s3.list_objects(bucket, path) + if not objects: + return "Empty" + + # Sample up to 10 objects + sample = objects[:10] + tiers = set() + for obj in sample: + # Get storage class from object metadata + storage_class = obj.get("StorageClass", "STANDARD") + # For Azure, also check BlobTier + blob_tier = obj.get("BlobTier") + if blob_tier: + tiers.add(blob_tier) + else: + tiers.add(storage_class) + + # Return summary + if len(tiers) == 1: + tier = tiers.pop() + # Normalize tier names for display + tier_display = { + "GLACIER": "Archive", + "DEEP_ARCHIVE": "Archive", + "Archive": "Archive", + "STANDARD": "Hot", + "Hot": "Hot", + "Cool": "Cool", + "STANDARD_IA": "Cool", + "ARCHIVE": "Archive", + "COLDLINE": "Archive", + "NEARLINE": "Cool", + }.get(tier, tier) + return tier_display + elif len(tiers) > 1: + return "Mixed" + else: + return "Unknown" + except Exception as e: + self.loggit.debug("Error getting storage tier for %s/%s: %s", bucket, base_path, e) + return "N/A" + def _get_repositories_status(self) -> list: """Get status of all repositories.""" repos = [] try: all_repos = get_all_repos(self.client) + # Get mounted repos list once + mounted_repos = get_matching_repo_names( + self.client, self.settings.repo_name_prefix + ) + for repo in all_repos: # Check if repo is actually mounted in ES - mounted_repos = get_matching_repo_names( - self.client, self.settings.repo_name_prefix - ) is_mounted_in_es = repo.name in mounted_repos + # Get storage tier (sample blobs) + storage_tier = self._get_repo_storage_tier(repo.bucket, repo.base_path) + repos.append( { "name": repo.name, @@ -117,6 +190,7 @@ def _get_repositories_status(self) -> list: "end": repo.end.isoformat() if repo.end else None, "is_mounted": is_mounted_in_es, "thaw_state": repo.thaw_state, + "storage_tier": storage_tier, "thawed_at": ( repo.thawed_at.isoformat() if repo.thawed_at else None ), @@ -186,7 +260,6 @@ def _get_ilm_policies(self) -> list: policies.append( { "name": policy_name, - "phase": phase_name, "repository": snapshot_repo, "indices_count": len(in_use_by.get("indices", [])), "data_streams_count": len( @@ -258,11 +331,15 @@ def _display_rich( table.add_column("Date Range", style="white") table.add_column("Mounted", style="green") table.add_column("Thaw State", style="magenta") + table.add_column("Storage Tier", style="blue") for repo in sorted(display_repos, key=lambda x: x["name"]): date_range = "" if repo.get("start") and repo.get("end"): - date_range = f"{repo['start'][:10]} - {repo['end'][:10]}" + date_range = ( + f"{self._format_date_value(repo['start'])} - " + f"{self._format_date_value(repo['end'])}" + ) mounted_str = ( "[green]Yes[/green]" @@ -278,6 +355,17 @@ def _display_rich( "expired": "red", }.get(thaw_state, "white") + # Storage tier with color coding + storage_tier = repo.get("storage_tier", "N/A") + tier_color = { + "Archive": "blue", + "Hot": "green", + "Cool": "cyan", + "Mixed": "yellow", + "Empty": "dim", + "N/A": "dim", + }.get(storage_tier, "white") + table.add_row( repo["name"], repo.get("bucket", "N/A"), @@ -285,6 +373,7 @@ def _display_rich( date_range, mounted_str, f"[{state_color}]{thaw_state}[/{state_color}]", + f"[{tier_color}]{storage_tier}[/{tier_color}]", ) self.console.print(table) @@ -315,7 +404,8 @@ def _display_rich( date_range = "" if req.get("start_date") and req.get("end_date"): date_range = ( - f"{req['start_date'][:10]} - {req['end_date'][:10]}" + f"{self._format_date_value(req['start_date'])} - " + f"{self._format_date_value(req['end_date'])}" ) repos_str = ", ".join(req.get("repos", [])[:3]) @@ -324,7 +414,7 @@ def _display_rich( created = req.get("created_at", "N/A") if created and created != "N/A": - created = created[:16].replace("T", " ") + created = self._format_created_value(created) table.add_row( req.get("request_id", "N/A"), @@ -366,7 +456,6 @@ def _display_rich( if ilm_policies: table = Table(title="ILM Policies (referencing deepfreeze)") table.add_column("Policy Name", style="cyan") - table.add_column("Phase", style="yellow") table.add_column("Repository", style="green") table.add_column("Indices", style="white") table.add_column("Data Streams", style="white") @@ -375,7 +464,6 @@ def _display_rich( for policy in sorted(ilm_policies, key=lambda x: x["name"]): table.add_row( policy["name"], - policy["phase"], policy["repository"], str(policy["indices_count"]), str(policy["data_streams_count"]), diff --git a/packages/deepfreeze-core/deepfreeze_core/aws_client.py b/packages/deepfreeze-core/deepfreeze_core/aws_client.py new file mode 100644 index 0000000..8a1a86c --- /dev/null +++ b/packages/deepfreeze-core/deepfreeze_core/aws_client.py @@ -0,0 +1,533 @@ +""" +aws_client.py + +AWS S3 client implementation for the deepfreeze package. +""" + +import logging + +import boto3 +from botocore.exceptions import ClientError + +from deepfreeze_core.exceptions import ActionError +from deepfreeze_core.s3client import S3Client + + +class AwsS3Client(S3Client): + """ + An S3 client object for use with AWS. + + Credentials can be provided via constructor arguments or environment variables. + Constructor arguments take precedence over environment variables. + + Args: + region: AWS region (e.g., 'us-east-1') + profile: AWS profile name from ~/.aws/credentials + access_key_id: AWS access key ID + secret_access_key: AWS secret access key + + Environment variables (fallback): + AWS_DEFAULT_REGION: Region + AWS_PROFILE: Profile name + AWS_ACCESS_KEY_ID: Access key ID + AWS_SECRET_ACCESS_KEY: Secret access key + """ + + # Elasticsearch repository plugin information + ES_PLUGIN_NAME = "repository-s3" + ES_PLUGIN_DISPLAY_NAME = "S3" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" + STORAGE_TYPE = "S3 bucket" + + # Elasticsearch keystore setup instructions + ES_KEYSTORE_INSTRUCTIONS = """Configure AWS credentials in Elasticsearch keystore: + bin/elasticsearch-keystore add s3.client.default.access_key + bin/elasticsearch-keystore add s3.client.default.secret_key + +Then restart Elasticsearch to apply the keystore changes.""" + + # Storage bucket creation error help + STORAGE_CREATION_HELP = """Possible solutions: + - Check AWS credentials and permissions + - Verify IAM policy allows s3:CreateBucket + - Check if bucket name is globally unique + - Verify AWS region settings + - Check AWS account limits for S3 buckets""" + + # Storage bucket deletion command template ({bucket} will be replaced) + STORAGE_DELETE_CMD = "aws s3 rb s3://{bucket} --force" + + def __init__( + self, + region: str = None, + profile: str = None, + access_key_id: str = None, + secret_access_key: str = None, + ) -> None: + self.loggit = logging.getLogger("deepfreeze.s3client") + try: + # Build session/client kwargs based on provided credentials + session_kwargs = {} + client_kwargs = {} + + if profile: + session_kwargs["profile_name"] = profile + self.loggit.debug("Using AWS profile: %s (source: config)", profile) + + if region: + session_kwargs["region_name"] = region + self.loggit.debug("Using AWS region: %s (source: config)", region) + + if access_key_id and secret_access_key: + session_kwargs["aws_access_key_id"] = access_key_id + session_kwargs["aws_secret_access_key"] = secret_access_key + self.loggit.debug("Using explicit AWS credentials (source: config)") + + # Create session and client + if session_kwargs: + session = boto3.Session(**session_kwargs) + self.client = session.client("s3", **client_kwargs) + else: + self.client = boto3.client("s3", **client_kwargs) + self.loggit.debug( + "Using default AWS credentials (environment/instance)" + ) + + # Validate credentials by attempting a simple operation + self.loggit.debug("Validating AWS credentials") + self.client.list_buckets() + self.loggit.info("AWS S3 Client initialized successfully") + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + self.loggit.error( + "Failed to initialize AWS S3 Client: %s - %s", error_code, e + ) + if error_code in ["InvalidAccessKeyId", "SignatureDoesNotMatch"]: + raise ActionError( + "AWS credentials are invalid or not configured. " + "Check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY." + ) from e + elif error_code == "AccessDenied": + raise ActionError( + "AWS credentials do not have sufficient permissions. " + "Minimum required: s3:ListAllMyBuckets" + ) from e + raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e + except Exception as e: + self.loggit.error( + "Failed to initialize AWS S3 Client: %s", e, exc_info=True + ) + raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e + + def test_connection(self) -> bool: + """ + Test S3 connection and validate credentials. + + :return: True if credentials are valid and S3 is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing S3 connection") + self.client.list_buckets() + return True + except ClientError as e: + self.loggit.error("S3 connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + self.loggit.info(f"Creating bucket: {bucket_name}") + if self.bucket_exists(bucket_name): + self.loggit.info(f"Bucket {bucket_name} already exists") + raise ActionError(f"Bucket {bucket_name} already exists") + try: + # Add region handling for bucket creation + # Get the region from the client configuration + region = self.client.meta.region_name + self.loggit.debug(f"Creating bucket in region: {region}") + + # AWS requires LocationConstraint for all regions except us-east-1 + if region and region != "us-east-1": + self.client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={"LocationConstraint": region}, + ) + self.loggit.info( + f"Successfully created bucket {bucket_name} in region {region}" + ) + else: + self.client.create_bucket(Bucket=bucket_name) + self.loggit.info( + f"Successfully created bucket {bucket_name} in us-east-1" + ) + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + self.loggit.error( + f"Error creating bucket {bucket_name}: {error_code} - {e}" + ) + raise ActionError(f"Error creating bucket {bucket_name}: {e}") from e + + def bucket_exists(self, bucket_name: str) -> bool: + self.loggit.debug(f"Checking if bucket {bucket_name} exists") + try: + self.client.head_bucket(Bucket=bucket_name) + self.loggit.debug(f"Bucket {bucket_name} exists") + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + self.loggit.debug(f"Bucket {bucket_name} does not exist") + return False + else: + self.loggit.error( + "Error checking bucket existence for %s: %s", bucket_name, e + ) + raise ActionError(e) from e + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[dict], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Restores objects from Glacier storage class back to an instant access tier. + + Args: + bucket_name (str): The name of the bucket + base_path (str): The base path (prefix) of the objects to thaw + object_keys (list[dict]): A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.) + restore_days (int): The number of days to keep the object restored + retrieval_tier (str): The retrieval tier to use + + Returns: + None + """ + self.loggit.info( + "Starting thaw operation - bucket: %s, base_path: %s, objects: %d, restore_days: %d, tier: %s", + bucket_name, + base_path, + len(object_keys), + restore_days, + retrieval_tier, + ) + + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, obj in enumerate(object_keys, 1): + # Extract key from object metadata dict + key = obj.get("Key") if isinstance(obj, dict) else obj + + if not key.startswith(base_path): + skipped_count += 1 + continue # Skip objects outside the base path + + # Get storage class from object metadata (if available) or fetch it + if isinstance(obj, dict) and "StorageClass" in obj: + storage_class = obj.get("StorageClass", "") + else: + try: + response = self.client.head_object(Bucket=bucket_name, Key=key) + storage_class = response.get("StorageClass", "") + except Exception as e: + error_count += 1 + self.loggit.error( + "Error getting metadata for object %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + continue + + try: + if storage_class in ["GLACIER", "DEEP_ARCHIVE", "GLACIER_IR"]: + self.loggit.debug( + "Restoring object %d/%d: %s from %s", + idx, + len(object_keys), + key, + storage_class, + ) + self.client.restore_object( + Bucket=bucket_name, + Key=key, + RestoreRequest={ + "Days": restore_days, + "GlacierJobParameters": {"Tier": retrieval_tier}, + }, + ) + restored_count += 1 + else: + self.loggit.debug( + "Skipping object %d/%d: %s (storage class: %s, not in Glacier)", + idx, + len(object_keys), + key, + storage_class, + ) + skipped_count += 1 + + except Exception as e: + error_count += 1 + self.loggit.error( + "Error restoring object %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + + # Log summary + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys), + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Moves objects back to a Glacier-tier storage class. + + Args: + bucket_name (str): The name of the bucket + path (str): The path to the objects to refreeze + storage_class (str): The storage class to move the objects to + + Returns: + None + """ + self.loggit.info( + "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class, + ) + + refrozen_count = 0 + skipped_count = 0 + error_count = 0 + + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=path) + + for page_num, page in enumerate(pages, 1): + if "Contents" in page: + page_objects = len(page["Contents"]) + self.loggit.debug( + "Processing page %d with %d objects", page_num, page_objects + ) + + for obj_num, obj in enumerate(page["Contents"], 1): + key = obj["Key"] + current_storage = obj.get("StorageClass", "STANDARD") + + # Skip objects already in the target storage class + if current_storage == storage_class: + self.loggit.debug( + "Skipping object %s - already in %s", + key, + current_storage, + ) + skipped_count += 1 + continue + + try: + # Copy the object with a new storage class + self.loggit.debug( + "Refreezing object %d/%d in page %d: %s (from %s to %s)", + obj_num, + page_objects, + page_num, + key, + current_storage, + storage_class, + ) + self.client.copy_object( + Bucket=bucket_name, + CopySource={"Bucket": bucket_name, "Key": key}, + Key=key, + StorageClass=storage_class, + ) + refrozen_count += 1 + + except Exception as e: + error_count += 1 + self.loggit.error( + "Error refreezing object %s: %s (type: %s)", + key, + str(e), + type(e).__name__, + exc_info=True, + ) + + # Log summary + self.loggit.info( + "Refreeze operation completed - changed: %d, skipped (already archived): %d, errors: %d", + refrozen_count, + skipped_count, + error_count, + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: + """ + List objects in a bucket with a given prefix. + + Args: + bucket_name (str): The name of the bucket to list objects from. + prefix (str): The prefix to use when listing objects. + + Returns: + list[dict]: A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.). + """ + self.loggit.info( + f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" + ) + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + objects = [] + + for page in pages: + if "Contents" in page: + for obj in page["Contents"]: + objects.append(obj) + + return objects + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete a bucket with the given name. + + Args: + bucket_name (str): The name of the bucket to delete. + force (bool): If True, empty the bucket before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting bucket: {bucket_name}") + try: + # If force=True, empty the bucket first + if force: + self.loggit.info(f"Emptying bucket {bucket_name} before deletion") + try: + # List and delete all objects + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name) + + for page in pages: + if "Contents" in page: + objects = [{"Key": obj["Key"]} for obj in page["Contents"]] + if objects: + self.client.delete_objects( + Bucket=bucket_name, Delete={"Objects": objects} + ) + self.loggit.debug( + f"Deleted {len(objects)} objects from {bucket_name}" + ) + except ClientError as e: + if e.response["Error"]["Code"] != "NoSuchBucket": + self.loggit.warning(f"Error emptying bucket {bucket_name}: {e}") + + self.client.delete_bucket(Bucket=bucket_name) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Put an object in a bucket. + + Args: + bucket_name (str): The name of the bucket to put the object in. + key (str): The key of the object to put. + body (str): The body of the object to put. + + Returns: + None + """ + self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") + try: + self.client.put_object(Bucket=bucket_name, Key=key, Body=body) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all buckets. + + Returns: + list[str]: A list of bucket names. + """ + self.loggit.info("Listing buckets") + try: + response = self.client.list_buckets() + buckets = response.get("Buckets", []) + bucket_names = [bucket["Name"] for bucket in buckets] + if prefix: + bucket_names = [ + name for name in bucket_names if name.startswith(prefix) + ] + return bucket_names + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for an object without downloading it. + + Args: + bucket_name (str): The name of the bucket. + key (str): The object key. + + Returns: + dict: Object metadata including Restore status if applicable. + """ + self.loggit.debug(f"Getting metadata for s3://{bucket_name}/{key}") + try: + response = self.client.head_object(Bucket=bucket_name, Key=key) + return response + except ClientError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") from e + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy an object from one bucket to another. + + Args: + Bucket (str): The name of the destination bucket. + Key (str): The key for the copied object. + CopySource (dict[str, str]): The source bucket and key. + StorageClass (str): The storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying object {Key} to bucket {Bucket}") + try: + self.client.copy_object( + Bucket=Bucket, + CopySource=CopySource, + Key=Key, + StorageClass=StorageClass, + ) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) from e diff --git a/packages/deepfreeze-core/deepfreeze_core/azure_client.py b/packages/deepfreeze-core/deepfreeze_core/azure_client.py new file mode 100644 index 0000000..f616e78 --- /dev/null +++ b/packages/deepfreeze-core/deepfreeze_core/azure_client.py @@ -0,0 +1,672 @@ +""" +azure_client.py + +Azure Blob Storage client implementation for the deepfreeze package. +Maps S3 concepts to Azure equivalents: +- S3 Bucket -> Azure Blob Container +- S3 Object -> Azure Blob +- Glacier/Deep Archive -> Azure Archive tier +- S3 restore -> Azure blob rehydration +""" + +import logging +import os + +from azure.core.exceptions import ( + AzureError, + ResourceExistsError, + ResourceNotFoundError, +) +from azure.storage.blob import BlobServiceClient, StandardBlobTier + +from deepfreeze_core.exceptions import ActionError +from deepfreeze_core.s3client import S3Client + + +class AzureBlobClient(S3Client): + """ + Azure Blob Storage client implementing the S3Client interface. + + Credentials can be provided via constructor arguments or environment variables. + Constructor arguments take precedence over environment variables. + + Args: + connection_string: Azure Storage connection string + account_name: Azure Storage account name (used with account_key) + account_key: Azure Storage account key (used with account_name) + + Environment variables (fallback): + AZURE_STORAGE_CONNECTION_STRING: Connection string + AZURE_STORAGE_ACCOUNT: Account name + AZURE_STORAGE_KEY: Account key + """ + + # Elasticsearch repository plugin information + ES_PLUGIN_NAME = "repository-azure" + ES_PLUGIN_DISPLAY_NAME = "Azure" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-azure.html" + STORAGE_TYPE = "Azure container" + + # Elasticsearch keystore setup instructions + ES_KEYSTORE_INSTRUCTIONS = """Configure Azure credentials in Elasticsearch keystore: + bin/elasticsearch-keystore add azure.client.default.account + bin/elasticsearch-keystore add azure.client.default.key + +Then restart Elasticsearch to apply the keystore changes.""" + + # Storage bucket creation error help + STORAGE_CREATION_HELP = """Possible solutions: + - Check Azure credentials (connection string or account name + key) + - Verify the storage account exists and is accessible + - Check if container name is valid (lowercase, no underscores, 3-63 chars) + - Verify Azure RBAC permissions allow container creation""" + + # Storage bucket deletion command template ({bucket} will be replaced) + STORAGE_DELETE_CMD = "az storage container delete --name {bucket} --account-name " + + def __init__( + self, + connection_string: str = None, + account_name: str = None, + account_key: str = None, + ) -> None: + self.loggit = logging.getLogger("deepfreeze.azure_client") + self.account_name = None # Will be set during initialization + try: + # Priority: constructor args > environment variables + conn_str = connection_string or os.environ.get( + "AZURE_STORAGE_CONNECTION_STRING" + ) + acct_name = account_name or os.environ.get("AZURE_STORAGE_ACCOUNT") + acct_key = account_key or os.environ.get("AZURE_STORAGE_KEY") + + if conn_str: + self.service_client = BlobServiceClient.from_connection_string(conn_str) + # Extract account name from connection string + self.account_name = self._extract_account_name_from_conn_str(conn_str) + self.loggit.debug( + "Using connection string for auth (source: %s, account: %s)", + "config" if connection_string else "environment", + self.account_name, + ) + elif acct_name and acct_key: + account_url = f"https://{acct_name}.blob.core.windows.net" + self.service_client = BlobServiceClient( + account_url=account_url, credential=acct_key + ) + self.account_name = acct_name + self.loggit.debug( + "Using account name + key for auth (source: %s, account: %s)", + "config" if account_name else "environment", + self.account_name, + ) + else: + raise ActionError( + "Azure credentials not configured. Provide connection_string or " + "account_name + account_key in config, or set environment variables " + "AZURE_STORAGE_CONNECTION_STRING or AZURE_STORAGE_ACCOUNT + AZURE_STORAGE_KEY." + ) + + # Validate credentials + self.loggit.debug("Validating Azure credentials") + # Use results_per_page (azure-storage-blob 12.x parameter name) + list(self.service_client.list_containers(results_per_page=1)) + self.loggit.info("Azure Blob Storage Client initialized successfully") + + except AzureError as e: + self.loggit.error("Failed to initialize Azure Blob Storage Client: %s", e) + raise ActionError( + f"Failed to initialize Azure Blob Storage Client: {e}" + ) from e + except Exception as e: + self.loggit.error( + "Failed to initialize Azure Blob Storage Client: %s", e, exc_info=True + ) + raise ActionError( + f"Failed to initialize Azure Blob Storage Client: {e}" + ) from e + + def test_connection(self) -> bool: + """ + Test Azure connection and validate credentials. + + :return: True if credentials are valid and Azure is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing Azure connection") + list(self.service_client.list_containers(results_per_page=1)) + return True + except AzureError as e: + self.loggit.error("Azure connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + """Create an Azure Blob container (equivalent to S3 bucket).""" + self.loggit.info( + "Creating container: %s in storage account: %s", + bucket_name, + self.account_name, + ) + if self.bucket_exists(bucket_name): + self.loggit.info( + "Container %s already exists in storage account %s", + bucket_name, + self.account_name, + ) + raise ActionError( + f"Container {bucket_name} already exists in storage account {self.account_name}" + ) + try: + self.service_client.create_container(bucket_name) + self.loggit.info( + "Successfully created container %s in storage account %s", + bucket_name, + self.account_name, + ) + except ResourceExistsError as e: + raise ActionError( + f"Container {bucket_name} already exists in storage account {self.account_name}" + ) from e + except AzureError as e: + self.loggit.error( + "Error creating container %s in storage account %s: %s", + bucket_name, + self.account_name, + e, + ) + raise ActionError( + f"Error creating container {bucket_name} in storage account {self.account_name}: {e}" + ) from e + + def bucket_exists(self, bucket_name: str) -> bool: + """Check if an Azure Blob container exists.""" + self.loggit.debug( + "Checking if container %s exists in storage account %s", + bucket_name, + self.account_name, + ) + try: + container_client = self.service_client.get_container_client(bucket_name) + container_client.get_container_properties() + self.loggit.debug( + "Container %s exists in storage account %s", + bucket_name, + self.account_name, + ) + return True + except ResourceNotFoundError: + self.loggit.debug( + "Container %s does not exist in storage account %s", + bucket_name, + self.account_name, + ) + return False + except AzureError as e: + self.loggit.error( + "Error checking container existence for %s in storage account %s: %s", + bucket_name, + self.account_name, + e, + ) + raise ActionError(e) from e + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[dict], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Rehydrate blobs from Archive tier. + + Azure rehydration is different from S3 Glacier restore: + - Uses set_standard_blob_tier() with rehydrate_priority + - Tier options: 'Hot', 'Cool' (Archive cannot be read directly) + - Rehydrate priorities: 'Standard' (up to 15 hours), 'High' (under 1 hour) + - Note: restore_days is not used in Azure - rehydration changes the tier permanently + + Args: + bucket_name (str): The name of the container + base_path (str): The base path (prefix) of the blobs to thaw + object_keys (list[dict]): A list of blob metadata dictionaries + restore_days (int): Not used in Azure (kept for interface compatibility) + retrieval_tier (str): 'Standard', 'Bulk', or 'Expedited' (mapped to Azure priorities) + + Returns: + None + """ + self.loggit.info( + "Starting thaw operation - container: %s, base_path: %s, objects: %d, tier: %s", + bucket_name, + base_path, + len(object_keys), + retrieval_tier, + ) + + # Map S3 retrieval tier to Azure rehydrate priority + priority_map = { + "Standard": "Standard", + "Bulk": "Standard", # Azure doesn't have Bulk, use Standard + "Expedited": "High", + } + rehydrate_priority = priority_map.get(retrieval_tier, "Standard") + + container_client = self.service_client.get_container_client(bucket_name) + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, obj in enumerate(object_keys, 1): + key = obj.get("Key") if isinstance(obj, dict) else obj + + if not key.startswith(base_path): + skipped_count += 1 + continue + + try: + blob_client = container_client.get_blob_client(key) + properties = blob_client.get_blob_properties() + current_tier = properties.blob_tier + + if current_tier == "Archive": + self.loggit.debug( + "Rehydrating blob %d/%d: %s from Archive (priority: %s)", + idx, + len(object_keys), + key, + rehydrate_priority, + ) + # Rehydrate to Hot tier + blob_client.set_standard_blob_tier( + StandardBlobTier.HOT, rehydrate_priority=rehydrate_priority + ) + restored_count += 1 + else: + self.loggit.debug( + "Skipping blob %d/%d: %s (tier: %s, not Archive)", + idx, + len(object_keys), + key, + current_tier, + ) + skipped_count += 1 + + except AzureError as e: + error_count += 1 + self.loggit.error( + "Error rehydrating blob %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys), + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Move blobs back to Archive tier. + + Maps S3 storage classes to Azure tiers: + - GLACIER -> Archive + - DEEP_ARCHIVE -> Archive (Azure has only one archive tier) + - GLACIER_IR -> Cool (closest equivalent) + + Args: + bucket_name (str): The name of the container + path (str): The path (prefix) to the blobs to refreeze + storage_class (str): The S3-style storage class to move blobs to + + Returns: + None + """ + self.loggit.info( + "Starting refreeze operation - container: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class, + ) + + # Map S3 storage class to Azure tier + tier_map = { + "GLACIER": StandardBlobTier.ARCHIVE, + "DEEP_ARCHIVE": StandardBlobTier.ARCHIVE, + "GLACIER_IR": StandardBlobTier.COOL, + } + target_tier = tier_map.get(storage_class, StandardBlobTier.ARCHIVE) + + container_client = self.service_client.get_container_client(bucket_name) + refrozen_count = 0 + skipped_count = 0 + error_count = 0 + + # Map target tier to string for comparison + target_tier_str = str(target_tier).replace("StandardBlobTier.", "") + + # List blobs with prefix + blobs = container_client.list_blobs(name_starts_with=path) + + for blob in blobs: + current_tier = blob.blob_tier + + # Skip blobs already in the target tier + if current_tier == target_tier_str: + self.loggit.debug( + "Skipping blob %s - already in %s tier", + blob.name, + current_tier, + ) + skipped_count += 1 + continue + + try: + blob_client = container_client.get_blob_client(blob.name) + self.loggit.debug( + "Changing tier for blob: %s (from %s to %s)", + blob.name, + current_tier, + target_tier_str, + ) + blob_client.set_standard_blob_tier(target_tier) + refrozen_count += 1 + except AzureError as e: + error_count += 1 + self.loggit.error( + "Error changing tier for blob %s: %s (type: %s)", + blob.name, + str(e), + type(e).__name__, + exc_info=True, + ) + + self.loggit.info( + "Refreeze operation completed - changed: %d, skipped (already archived): %d, errors: %d", + refrozen_count, + skipped_count, + error_count, + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: + """ + List blobs in a container with a given prefix. + + Args: + bucket_name (str): The name of the container to list blobs from. + prefix (str): The prefix to use when listing blobs. + + Returns: + list[dict]: A list of blob metadata dictionaries with S3-compatible keys. + """ + self.loggit.info( + f"Listing blobs in container: {bucket_name} with prefix: {prefix}" + ) + + container_client = self.service_client.get_container_client(bucket_name) + objects = [] + + try: + blobs = container_client.list_blobs(name_starts_with=prefix) + for blob in blobs: + # Map Azure properties to S3-like structure for compatibility + objects.append( + { + "Key": blob.name, + "Size": blob.size, + "LastModified": blob.last_modified, + "StorageClass": self._map_azure_tier_to_s3(blob.blob_tier), + # Azure-specific + "BlobTier": blob.blob_tier, + "ArchiveStatus": getattr(blob, "archive_status", None), + } + ) + return objects + except AzureError as e: + self.loggit.error("Error listing blobs: %s", e) + raise ActionError(f"Error listing blobs: {e}") from e + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete an Azure Blob container. + + Args: + bucket_name (str): The name of the container to delete. + force (bool): If True, empty the container before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting container: {bucket_name}") + try: + container_client = self.service_client.get_container_client(bucket_name) + + if force: + self.loggit.info(f"Emptying container {bucket_name} before deletion") + blobs = container_client.list_blobs() + for blob in blobs: + container_client.delete_blob(blob.name) + self.loggit.debug(f"Deleted blob {blob.name}") + + container_client.delete_container() + self.loggit.info(f"Container {bucket_name} deleted successfully") + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Upload a blob to a container. + + Args: + bucket_name (str): The name of the container to put the blob in. + key (str): The key of the blob to put. + body (str): The body of the blob to put. + + Returns: + None + """ + self.loggit.info(f"Putting blob: {key} in container: {bucket_name}") + try: + container_client = self.service_client.get_container_client(bucket_name) + blob_client = container_client.get_blob_client(key) + blob_client.upload_blob(body, overwrite=True) + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all containers. + + Args: + prefix (str): Optional prefix to filter container names. + + Returns: + list[str]: A list of container names. + """ + self.loggit.info("Listing containers") + try: + containers = self.service_client.list_containers() + container_names = [c.name for c in containers] + if prefix: + container_names = [ + name for name in container_names if name.startswith(prefix) + ] + return container_names + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for a blob without downloading it. + + Args: + bucket_name (str): The name of the container. + key (str): The blob key. + + Returns: + dict: Blob metadata including Restore status if applicable. + """ + self.loggit.debug(f"Getting metadata for azure://{bucket_name}/{key}") + try: + container_client = self.service_client.get_container_client(bucket_name) + blob_client = container_client.get_blob_client(key) + properties = blob_client.get_blob_properties() + + # Map to S3-like response structure + response = { + "ContentLength": properties.size, + "ContentType": properties.content_settings.content_type, + "LastModified": properties.last_modified, + "ETag": properties.etag, + "StorageClass": self._map_azure_tier_to_s3(properties.blob_tier), + # Azure rehydration status maps to S3 Restore header + "Restore": self._format_restore_header(properties), + # Azure-specific + "BlobTier": properties.blob_tier, + "ArchiveStatus": getattr(properties, "archive_status", None), + } + return response + except ResourceNotFoundError as e: + self.loggit.error(f"Blob not found: {key}") + raise ActionError(f"Blob not found: {key}") from e + except AzureError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") from e + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy a blob with storage class change. + + Args: + Bucket (str): The name of the destination container. + Key (str): The key for the copied blob. + CopySource (dict[str, str]): The source container and key. + StorageClass (str): The S3-style storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying blob {Key} to container {Bucket}") + try: + source_container = CopySource["Bucket"] + source_key = CopySource["Key"] + + # Get source URL + source_container_client = self.service_client.get_container_client( + source_container + ) + source_blob_client = source_container_client.get_blob_client(source_key) + source_url = source_blob_client.url + + # Copy to destination + dest_container_client = self.service_client.get_container_client(Bucket) + dest_blob_client = dest_container_client.get_blob_client(Key) + + # Start copy operation + dest_blob_client.start_copy_from_url(source_url) + + # Set storage tier + tier_map = { + "GLACIER": StandardBlobTier.ARCHIVE, + "DEEP_ARCHIVE": StandardBlobTier.ARCHIVE, + "STANDARD": StandardBlobTier.HOT, + "STANDARD_IA": StandardBlobTier.COOL, + } + target_tier = tier_map.get(StorageClass, StandardBlobTier.ARCHIVE) + dest_blob_client.set_standard_blob_tier(target_tier) + + except AzureError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def _map_azure_tier_to_s3(self, azure_tier: str) -> str: + """ + Map Azure blob tier to S3 storage class for compatibility. + + Args: + azure_tier (str): The Azure blob tier. + + Returns: + str: The equivalent S3 storage class. + """ + tier_map = { + "Hot": "STANDARD", + "Cool": "STANDARD_IA", + "Cold": "ONEZONE_IA", + "Archive": "GLACIER", + } + return tier_map.get(azure_tier, "STANDARD") + + def _format_restore_header(self, properties) -> str: + """ + Format Azure rehydration status to S3-like Restore header. + + Args: + properties: The blob properties object. + + Returns: + str: S3-style restore header or None. + """ + archive_status = getattr(properties, "archive_status", None) + blob_tier = properties.blob_tier + + if blob_tier == "Archive": + if archive_status in [ + "rehydrate-pending-to-hot", + "rehydrate-pending-to-cool", + ]: + return 'ongoing-request="true"' + return None # Not restored + elif blob_tier in ("Hot", "Cool") and archive_status: + # Was rehydrated from archive + return 'ongoing-request="false"' + return None + + def _extract_account_name_from_conn_str(self, conn_str: str) -> str: + """ + Extract the account name from an Azure connection string. + + Args: + conn_str (str): The Azure Storage connection string. + + Returns: + str: The storage account name, or "unknown" if not found. + """ + try: + for part in conn_str.split(";"): + if part.startswith("AccountName="): + return part.split("=", 1)[1] + except Exception: + pass + return "unknown" + + def get_storage_delete_cmd(self, bucket_name: str) -> str: + """ + Get the provider-specific command to delete a storage container. + + Args: + bucket_name (str): The name of the container to delete. + + Returns: + str: The command string with bucket and account name filled in. + """ + return f"az storage container delete --name {bucket_name} --account-name {self.account_name}" diff --git a/packages/deepfreeze-core/deepfreeze_core/constants.py b/packages/deepfreeze-core/deepfreeze_core/constants.py index b8f4510..5570d16 100644 --- a/packages/deepfreeze-core/deepfreeze_core/constants.py +++ b/packages/deepfreeze-core/deepfreeze_core/constants.py @@ -9,15 +9,15 @@ SETTINGS_ID = "1" # Supported cloud providers -PROVIDERS = ["aws"] +PROVIDERS = ["aws", "azure", "gcp"] # Repository thaw lifecycle states THAW_STATE_ACTIVE = "active" # Active repository, never been through thaw lifecycle -THAW_STATE_FROZEN = "frozen" # In cold storage (Glacier), not currently accessible -THAW_STATE_THAWING = "thawing" # S3 restore in progress, waiting for retrieval -THAW_STATE_THAWED = "thawed" # S3 restore complete, mounted and in use +THAW_STATE_FROZEN = "frozen" # In cold storage (archive tier), not currently accessible +THAW_STATE_THAWING = "thawing" # Restore in progress, waiting for retrieval +THAW_STATE_THAWED = "thawed" # Restore complete, mounted and in use THAW_STATE_EXPIRED = ( - "expired" # S3 restore expired, reverted to Glacier, ready for cleanup + "expired" # Restore expired, reverted to archive tier, ready for cleanup ) THAW_STATES = [ diff --git a/packages/deepfreeze-core/deepfreeze_core/esclient.py b/packages/deepfreeze-core/deepfreeze_core/esclient.py index 63ba9b7..bfd3737 100644 --- a/packages/deepfreeze-core/deepfreeze_core/esclient.py +++ b/packages/deepfreeze-core/deepfreeze_core/esclient.py @@ -185,6 +185,102 @@ def load_config_from_yaml(config_path: str) -> dict: raise ActionError(f"Error reading configuration file: {e}") from e +def load_storage_config(config_path: str) -> dict: + """ + Load storage provider configuration from a YAML file. + + The YAML file should include a 'storage' section with provider credentials: + + ```yaml + storage: + # AWS S3 configuration + aws: + region: us-east-1 + profile: my-profile # Optional: use named profile + # Or explicit credentials: + # access_key_id: AKIA... + # secret_access_key: ... + + # Azure Blob Storage configuration + azure: + connection_string: "DefaultEndpointsProtocol=https;AccountName=..." + # Or account name + key: + # account_name: mystorageaccount + # account_key: ... + + # Google Cloud Storage configuration + gcp: + project: my-gcp-project + credentials_file: /path/to/service-account.json + location: US + ``` + + Args: + config_path: Path to the YAML configuration file + + Returns: + dict: Storage configuration with provider-specific credentials. + Returns empty dict if no storage section exists. + + Example return value: + { + 'aws': {'region': 'us-east-1', 'profile': 'my-profile'}, + 'azure': {'connection_string': '...'}, + 'gcp': {'project': 'my-project', 'credentials_file': '/path/to/creds.json'} + } + """ + loggit = logging.getLogger("deepfreeze.esclient") + loggit.debug("Loading storage configuration from: %s", config_path) + + config = load_config_from_yaml(config_path) + storage_config = config.get("storage", {}) + + if storage_config: + loggit.debug( + "Found storage configuration for providers: %s", + list(storage_config.keys()), + ) + else: + loggit.debug("No storage configuration found in config file") + + return storage_config + + +def get_storage_credentials(config_path: str, provider: str) -> dict: + """ + Get storage credentials for a specific provider from the config file. + + This is a convenience function that loads the storage config and returns + only the credentials for the specified provider. + + Args: + config_path: Path to the YAML configuration file + provider: The storage provider ('aws', 'azure', or 'gcp') + + Returns: + dict: Provider-specific credentials ready to pass to s3_client_factory. + Returns empty dict if no credentials found for the provider. + + Example: + >>> creds = get_storage_credentials('/path/to/config.yaml', 'azure') + >>> client = s3_client_factory('azure', **creds) + """ + loggit = logging.getLogger("deepfreeze.esclient") + + storage_config = load_storage_config(config_path) + provider_config = storage_config.get(provider, {}) + + if provider_config: + loggit.debug("Found credentials for provider: %s", provider) + else: + loggit.debug( + "No credentials found for provider %s, will use environment variables", + provider, + ) + + return provider_config + + def create_es_client_from_config(config_path: str) -> Elasticsearch: """ Create an Elasticsearch client from a YAML configuration file. diff --git a/packages/deepfreeze-core/deepfreeze_core/gcp_client.py b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py new file mode 100644 index 0000000..4cbc1cc --- /dev/null +++ b/packages/deepfreeze-core/deepfreeze_core/gcp_client.py @@ -0,0 +1,553 @@ +""" +gcp_client.py + +Google Cloud Storage client implementation for the deepfreeze package. +Maps S3 concepts to GCP equivalents: +- S3 Bucket -> GCS Bucket +- S3 Object -> GCS Blob +- Glacier/Deep Archive -> GCS Archive storage class +- Storage classes: STANDARD, NEARLINE, COLDLINE, ARCHIVE +""" + +import logging +import os + +from google.api_core.exceptions import Conflict, GoogleAPIError, NotFound +from google.cloud import storage + +from deepfreeze_core.exceptions import ActionError +from deepfreeze_core.s3client import S3Client + + +class GcpStorageClient(S3Client): + """ + Google Cloud Storage client implementing the S3Client interface. + + Credentials can be provided via constructor arguments or environment variables. + Constructor arguments take precedence over environment variables. + + Args: + project: GCP project ID + credentials_file: Path to service account JSON credentials file + location: Default location for bucket creation (default: US) + + Environment variables (fallback): + GOOGLE_CLOUD_PROJECT: Project ID + GOOGLE_APPLICATION_CREDENTIALS: Path to credentials file + GOOGLE_CLOUD_LOCATION: Default location for buckets + """ + + # Elasticsearch repository plugin information + ES_PLUGIN_NAME = "repository-gcs" + ES_PLUGIN_DISPLAY_NAME = "GCS" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html" + STORAGE_TYPE = "GCS bucket" + + # Elasticsearch keystore setup instructions + ES_KEYSTORE_INSTRUCTIONS = """Configure GCP credentials in Elasticsearch keystore: + bin/elasticsearch-keystore add-file gcs.client.default.credentials_file /path/to/service-account.json + +Then restart Elasticsearch to apply the keystore changes.""" + + # Storage bucket creation error help + STORAGE_CREATION_HELP = """Possible solutions: + - Check GCP credentials (service account JSON file) + - Verify the GCP project is correctly configured + - Check if bucket name is globally unique + - Verify IAM permissions allow storage.buckets.create""" + + # Storage bucket deletion command template ({bucket} will be replaced) + STORAGE_DELETE_CMD = "gcloud storage rm -r gs://{bucket}" + + def __init__( + self, + project: str = None, + credentials_file: str = None, + location: str = None, + ) -> None: + self.loggit = logging.getLogger("deepfreeze.gcp_client") + self.default_location = location or os.environ.get( + "GOOGLE_CLOUD_LOCATION", "US" + ) + + try: + # Priority: constructor args > environment variables + project_id = project or os.environ.get("GOOGLE_CLOUD_PROJECT") + creds_file = credentials_file or os.environ.get( + "GOOGLE_APPLICATION_CREDENTIALS" + ) + + if creds_file: + # Use explicit credentials file + self.client = storage.Client.from_service_account_json( + creds_file, project=project_id + ) + self.loggit.debug( + "Using credentials file: %s (source: %s)", + creds_file, + "config" if credentials_file else "environment", + ) + elif project_id: + # Use ADC with explicit project + self.client = storage.Client(project=project_id) + self.loggit.debug( + "Using project %s with ADC (source: %s)", + project_id, + "config" if project else "environment", + ) + else: + # Use default ADC + self.client = storage.Client() + self.loggit.debug("Using default Application Default Credentials") + + # Validate credentials by listing buckets (limited to 1) + self.loggit.debug("Validating GCP credentials") + list(self.client.list_buckets(max_results=1)) + self.loggit.info("GCP Storage Client initialized successfully") + + except GoogleAPIError as e: + self.loggit.error("Failed to initialize GCP Storage Client: %s", e) + raise ActionError(f"Failed to initialize GCP Storage Client: {e}") from e + except Exception as e: + self.loggit.error( + "Failed to initialize GCP Storage Client: %s", e, exc_info=True + ) + raise ActionError(f"Failed to initialize GCP Storage Client: {e}") from e + + def test_connection(self) -> bool: + """ + Test GCP connection and validate credentials. + + :return: True if credentials are valid and GCS is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing GCP connection") + list(self.client.list_buckets(max_results=1)) + return True + except GoogleAPIError as e: + self.loggit.error("GCP connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + """Create a GCS bucket.""" + self.loggit.info(f"Creating bucket: {bucket_name}") + if self.bucket_exists(bucket_name): + self.loggit.info(f"Bucket {bucket_name} already exists") + raise ActionError(f"Bucket {bucket_name} already exists") + try: + bucket = self.client.bucket(bucket_name) + self.client.create_bucket(bucket, location=self.default_location) + self.loggit.info( + f"Successfully created bucket {bucket_name} in location {self.default_location}" + ) + except Conflict as e: + raise ActionError(f"Bucket {bucket_name} already exists") from e + except GoogleAPIError as e: + self.loggit.error(f"Error creating bucket {bucket_name}: {e}") + raise ActionError(f"Error creating bucket {bucket_name}: {e}") from e + + def bucket_exists(self, bucket_name: str) -> bool: + """Check if a GCS bucket exists.""" + self.loggit.debug(f"Checking if bucket {bucket_name} exists") + try: + bucket = self.client.bucket(bucket_name) + bucket.reload() + self.loggit.debug(f"Bucket {bucket_name} exists") + return True + except NotFound: + self.loggit.debug(f"Bucket {bucket_name} does not exist") + return False + except GoogleAPIError as e: + self.loggit.error( + "Error checking bucket existence for %s: %s", bucket_name, e + ) + raise ActionError(e) from e + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[dict], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Move objects from Archive/Coldline to Standard storage class. + + Unlike AWS Glacier, GCS Archive objects are immediately accessible, + just with higher retrieval costs. "Thawing" means changing the + storage class to STANDARD for faster/cheaper access. + + Args: + bucket_name (str): The name of the bucket + base_path (str): The base path (prefix) of the objects to thaw + object_keys (list[dict]): A list of object metadata dictionaries + restore_days (int): Not used in GCP (kept for interface compatibility) + retrieval_tier (str): Not used in GCP (kept for interface compatibility) + + Returns: + None + """ + self.loggit.info( + "Starting thaw operation - bucket: %s, base_path: %s, objects: %d", + bucket_name, + base_path, + len(object_keys), + ) + + bucket = self.client.bucket(bucket_name) + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, obj in enumerate(object_keys, 1): + key = obj.get("Key") if isinstance(obj, dict) else obj + + if not key.startswith(base_path): + skipped_count += 1 + continue + + try: + blob = bucket.blob(key) + blob.reload() + current_class = blob.storage_class + + if current_class in ["ARCHIVE", "COLDLINE", "NEARLINE"]: + self.loggit.debug( + "Thawing blob %d/%d: %s from %s to STANDARD", + idx, + len(object_keys), + key, + current_class, + ) + blob.update_storage_class("STANDARD") + restored_count += 1 + else: + self.loggit.debug( + "Skipping blob %d/%d: %s (storage class: %s)", + idx, + len(object_keys), + key, + current_class, + ) + skipped_count += 1 + + except GoogleAPIError as e: + error_count += 1 + self.loggit.error( + "Error thawing blob %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__, + ) + + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys), + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Move objects to Archive storage class. + + Maps S3 storage classes to GCS: + - GLACIER -> ARCHIVE + - DEEP_ARCHIVE -> ARCHIVE + - GLACIER_IR -> COLDLINE + + Args: + bucket_name (str): The name of the bucket + path (str): The path (prefix) to the objects to refreeze + storage_class (str): The S3-style storage class to move objects to + + Returns: + None + """ + self.loggit.info( + "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class, + ) + + # Map S3 storage class to GCS storage class + class_map = { + "GLACIER": "ARCHIVE", + "DEEP_ARCHIVE": "ARCHIVE", + "GLACIER_IR": "COLDLINE", + } + target_class = class_map.get(storage_class, "ARCHIVE") + + bucket = self.client.bucket(bucket_name) + refrozen_count = 0 + skipped_count = 0 + error_count = 0 + + # List blobs with prefix + blobs = bucket.list_blobs(prefix=path) + + for blob in blobs: + current_class = blob.storage_class + + # Skip blobs already in the target storage class + if current_class == target_class: + self.loggit.debug( + "Skipping blob %s - already in %s", + blob.name, + current_class, + ) + skipped_count += 1 + continue + + try: + self.loggit.debug( + "Refreezing blob: %s (from %s to %s)", + blob.name, + current_class, + target_class, + ) + blob.update_storage_class(target_class) + refrozen_count += 1 + except GoogleAPIError as e: + error_count += 1 + self.loggit.error( + "Error refreezing blob %s: %s (type: %s)", + blob.name, + str(e), + type(e).__name__, + exc_info=True, + ) + + self.loggit.info( + "Refreeze operation completed - changed: %d, skipped (already archived): %d, errors: %d", + refrozen_count, + skipped_count, + error_count, + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: + """ + List objects in a bucket with a given prefix. + + Args: + bucket_name (str): The name of the bucket to list objects from. + prefix (str): The prefix to use when listing objects. + + Returns: + list[dict]: A list of object metadata dictionaries with S3-compatible keys. + """ + self.loggit.info( + f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" + ) + + bucket = self.client.bucket(bucket_name) + objects = [] + + try: + blobs = bucket.list_blobs(prefix=prefix) + for blob in blobs: + # Map GCS properties to S3-like structure for compatibility + objects.append( + { + "Key": blob.name, + "Size": blob.size, + "LastModified": blob.updated, + "StorageClass": self._map_gcs_class_to_s3(blob.storage_class), + # GCS-specific + "GcsStorageClass": blob.storage_class, + } + ) + return objects + except GoogleAPIError as e: + self.loggit.error("Error listing objects: %s", e) + raise ActionError(f"Error listing objects: {e}") from e + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete a GCS bucket. + + Args: + bucket_name (str): The name of the bucket to delete. + force (bool): If True, empty the bucket before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting bucket: {bucket_name}") + try: + bucket = self.client.bucket(bucket_name) + + if force: + self.loggit.info(f"Emptying bucket {bucket_name} before deletion") + blobs = bucket.list_blobs() + for blob in blobs: + blob.delete() + self.loggit.debug(f"Deleted blob {blob.name}") + + bucket.delete() + self.loggit.info(f"Bucket {bucket_name} deleted successfully") + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Upload an object to a bucket. + + Args: + bucket_name (str): The name of the bucket to put the object in. + key (str): The key of the object to put. + body (str): The body of the object to put. + + Returns: + None + """ + self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") + try: + bucket = self.client.bucket(bucket_name) + blob = bucket.blob(key) + blob.upload_from_string(body) + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all buckets. + + Args: + prefix (str): Optional prefix to filter bucket names. + + Returns: + list[str]: A list of bucket names. + """ + self.loggit.info("Listing buckets") + try: + buckets = self.client.list_buckets() + bucket_names = [b.name for b in buckets] + if prefix: + bucket_names = [ + name for name in bucket_names if name.startswith(prefix) + ] + return bucket_names + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for an object without downloading it. + + Args: + bucket_name (str): The name of the bucket. + key (str): The object key. + + Returns: + dict: Object metadata including storage class. + """ + self.loggit.debug(f"Getting metadata for gs://{bucket_name}/{key}") + try: + bucket = self.client.bucket(bucket_name) + blob = bucket.blob(key) + blob.reload() + + # Map to S3-like response structure + response = { + "ContentLength": blob.size, + "ContentType": blob.content_type, + "LastModified": blob.updated, + "ETag": blob.etag, + "StorageClass": self._map_gcs_class_to_s3(blob.storage_class), + # GCS doesn't have restore concept like S3 Glacier + # Archive objects are immediately accessible + "Restore": None, + # GCS-specific + "GcsStorageClass": blob.storage_class, + } + return response + except NotFound as e: + self.loggit.error(f"Object not found: {key}") + raise ActionError(f"Object not found: {key}") from e + except GoogleAPIError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") from e + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy an object with storage class change. + + Args: + Bucket (str): The name of the destination bucket. + Key (str): The key for the copied object. + CopySource (dict[str, str]): The source bucket and key. + StorageClass (str): The S3-style storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying object {Key} to bucket {Bucket}") + try: + source_bucket_name = CopySource["Bucket"] + source_key = CopySource["Key"] + + # Map S3 storage class to GCS + class_map = { + "GLACIER": "ARCHIVE", + "DEEP_ARCHIVE": "ARCHIVE", + "STANDARD": "STANDARD", + "STANDARD_IA": "NEARLINE", + } + target_class = class_map.get(StorageClass, "ARCHIVE") + + source_bucket = self.client.bucket(source_bucket_name) + source_blob = source_bucket.blob(source_key) + + dest_bucket = self.client.bucket(Bucket) + dest_blob = dest_bucket.blob(Key) + + # Copy the blob + token = None + while True: + token, _, _ = dest_blob.rewrite(source_blob, token=token) + if token is None: + break + + # Update storage class if different from default + if target_class != "STANDARD": + dest_blob.update_storage_class(target_class) + + except GoogleAPIError as e: + self.loggit.error(e) + raise ActionError(e) from e + + def _map_gcs_class_to_s3(self, gcs_class: str) -> str: + """ + Map GCS storage class to S3 storage class for compatibility. + + Args: + gcs_class (str): The GCS storage class. + + Returns: + str: The equivalent S3 storage class. + """ + class_map = { + "STANDARD": "STANDARD", + "NEARLINE": "STANDARD_IA", + "COLDLINE": "ONEZONE_IA", + "ARCHIVE": "GLACIER", + } + return class_map.get(gcs_class, "STANDARD") diff --git a/packages/deepfreeze-core/deepfreeze_core/s3client.py b/packages/deepfreeze-core/deepfreeze_core/s3client.py index d8a55d2..d22c91b 100644 --- a/packages/deepfreeze-core/deepfreeze_core/s3client.py +++ b/packages/deepfreeze-core/deepfreeze_core/s3client.py @@ -6,12 +6,6 @@ """ import abc -import logging - -import boto3 -from botocore.exceptions import ClientError - -from deepfreeze_core.exceptions import ActionError class S3Client(metaclass=abc.ABCMeta): @@ -186,451 +180,31 @@ def copy_object( return -class AwsS3Client(S3Client): - """ - An S3 client object for use with AWS. - """ - - def __init__(self) -> None: - self.loggit = logging.getLogger("deepfreeze.s3client") - try: - self.client = boto3.client("s3") - # Validate credentials by attempting a simple operation - self.loggit.debug("Validating AWS credentials") - self.client.list_buckets() - self.loggit.info("AWS S3 Client initialized successfully") - except ClientError as e: - error_code = e.response.get("Error", {}).get("Code", "Unknown") - self.loggit.error( - "Failed to initialize AWS S3 Client: %s - %s", error_code, e - ) - if error_code in ["InvalidAccessKeyId", "SignatureDoesNotMatch"]: - raise ActionError( - "AWS credentials are invalid or not configured. " - "Check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY." - ) from e - elif error_code == "AccessDenied": - raise ActionError( - "AWS credentials do not have sufficient permissions. " - "Minimum required: s3:ListAllMyBuckets" - ) from e - raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e - except Exception as e: - self.loggit.error( - "Failed to initialize AWS S3 Client: %s", e, exc_info=True - ) - raise ActionError(f"Failed to initialize AWS S3 Client: {e}") from e - - def test_connection(self) -> bool: - """ - Test S3 connection and validate credentials. - - :return: True if credentials are valid and S3 is accessible - :rtype: bool - """ - try: - self.loggit.debug("Testing S3 connection") - self.client.list_buckets() - return True - except ClientError as e: - self.loggit.error("S3 connection test failed: %s", e) - return False - - def create_bucket(self, bucket_name: str) -> None: - self.loggit.info(f"Creating bucket: {bucket_name}") - if self.bucket_exists(bucket_name): - self.loggit.info(f"Bucket {bucket_name} already exists") - raise ActionError(f"Bucket {bucket_name} already exists") - try: - # Add region handling for bucket creation - # Get the region from the client configuration - region = self.client.meta.region_name - self.loggit.debug(f"Creating bucket in region: {region}") - - # AWS requires LocationConstraint for all regions except us-east-1 - if region and region != "us-east-1": - self.client.create_bucket( - Bucket=bucket_name, - CreateBucketConfiguration={"LocationConstraint": region}, - ) - self.loggit.info( - f"Successfully created bucket {bucket_name} in region {region}" - ) - else: - self.client.create_bucket(Bucket=bucket_name) - self.loggit.info( - f"Successfully created bucket {bucket_name} in us-east-1" - ) - except ClientError as e: - error_code = e.response.get("Error", {}).get("Code", "Unknown") - self.loggit.error( - f"Error creating bucket {bucket_name}: {error_code} - {e}" - ) - raise ActionError(f"Error creating bucket {bucket_name}: {e}") from e - - def bucket_exists(self, bucket_name: str) -> bool: - self.loggit.debug(f"Checking if bucket {bucket_name} exists") - try: - self.client.head_bucket(Bucket=bucket_name) - self.loggit.debug(f"Bucket {bucket_name} exists") - return True - except ClientError as e: - if e.response["Error"]["Code"] == "404": - self.loggit.debug(f"Bucket {bucket_name} does not exist") - return False - else: - self.loggit.error( - "Error checking bucket existence for %s: %s", bucket_name, e - ) - raise ActionError(e) from e - - def thaw( - self, - bucket_name: str, - base_path: str, - object_keys: list[dict], - restore_days: int = 7, - retrieval_tier: str = "Standard", - ) -> None: - """ - Restores objects from Glacier storage class back to an instant access tier. - - Args: - bucket_name (str): The name of the bucket - base_path (str): The base path (prefix) of the objects to thaw - object_keys (list[dict]): A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.) - restore_days (int): The number of days to keep the object restored - retrieval_tier (str): The retrieval tier to use - - Returns: - None - """ - self.loggit.info( - "Starting thaw operation - bucket: %s, base_path: %s, objects: %d, restore_days: %d, tier: %s", - bucket_name, - base_path, - len(object_keys), - restore_days, - retrieval_tier, - ) - - restored_count = 0 - skipped_count = 0 - error_count = 0 - - for idx, obj in enumerate(object_keys, 1): - # Extract key from object metadata dict - key = obj.get("Key") if isinstance(obj, dict) else obj - - if not key.startswith(base_path): - skipped_count += 1 - continue # Skip objects outside the base path - - # Get storage class from object metadata (if available) or fetch it - if isinstance(obj, dict) and "StorageClass" in obj: - storage_class = obj.get("StorageClass", "") - else: - try: - response = self.client.head_object(Bucket=bucket_name, Key=key) - storage_class = response.get("StorageClass", "") - except Exception as e: - error_count += 1 - self.loggit.error( - "Error getting metadata for object %d/%d (%s): %s (type: %s)", - idx, - len(object_keys), - key, - str(e), - type(e).__name__, - ) - continue - - try: - if storage_class in ["GLACIER", "DEEP_ARCHIVE", "GLACIER_IR"]: - self.loggit.debug( - "Restoring object %d/%d: %s from %s", - idx, - len(object_keys), - key, - storage_class, - ) - self.client.restore_object( - Bucket=bucket_name, - Key=key, - RestoreRequest={ - "Days": restore_days, - "GlacierJobParameters": {"Tier": retrieval_tier}, - }, - ) - restored_count += 1 - else: - self.loggit.debug( - "Skipping object %d/%d: %s (storage class: %s, not in Glacier)", - idx, - len(object_keys), - key, - storage_class, - ) - skipped_count += 1 - - except Exception as e: - error_count += 1 - self.loggit.error( - "Error restoring object %d/%d (%s): %s (type: %s)", - idx, - len(object_keys), - key, - str(e), - type(e).__name__, - ) - - # Log summary - self.loggit.info( - "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", - restored_count, - skipped_count, - error_count, - len(object_keys), - ) - - def refreeze( - self, bucket_name: str, path: str, storage_class: str = "GLACIER" - ) -> None: - """ - Moves objects back to a Glacier-tier storage class. - - Args: - bucket_name (str): The name of the bucket - path (str): The path to the objects to refreeze - storage_class (str): The storage class to move the objects to - - Returns: - None - """ - self.loggit.info( - "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", - bucket_name, - path, - storage_class, - ) - - refrozen_count = 0 - error_count = 0 - - paginator = self.client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name, Prefix=path) - - for page_num, page in enumerate(pages, 1): - if "Contents" in page: - page_objects = len(page["Contents"]) - self.loggit.debug( - "Processing page %d with %d objects", page_num, page_objects - ) - - for obj_num, obj in enumerate(page["Contents"], 1): - key = obj["Key"] - current_storage = obj.get("StorageClass", "STANDARD") - - try: - # Copy the object with a new storage class - self.loggit.debug( - "Refreezing object %d/%d in page %d: %s (from %s to %s)", - obj_num, - page_objects, - page_num, - key, - current_storage, - storage_class, - ) - self.client.copy_object( - Bucket=bucket_name, - CopySource={"Bucket": bucket_name, "Key": key}, - Key=key, - StorageClass=storage_class, - ) - refrozen_count += 1 - - except Exception as e: - error_count += 1 - self.loggit.error( - "Error refreezing object %s: %s (type: %s)", - key, - str(e), - type(e).__name__, - exc_info=True, - ) - - # Log summary - self.loggit.info( - "Refreeze operation completed - refrozen: %d, errors: %d", - refrozen_count, - error_count, - ) - - def list_objects(self, bucket_name: str, prefix: str) -> list[dict]: - """ - List objects in a bucket with a given prefix. - - Args: - bucket_name (str): The name of the bucket to list objects from. - prefix (str): The prefix to use when listing objects. - - Returns: - list[dict]: A list of object metadata dictionaries (each containing 'Key', 'StorageClass', etc.). - """ - self.loggit.info( - f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" - ) - paginator = self.client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) - objects = [] - - for page in pages: - if "Contents" in page: - for obj in page["Contents"]: - objects.append(obj) - - return objects - - def delete_bucket(self, bucket_name: str, force: bool = False) -> None: - """ - Delete a bucket with the given name. - - Args: - bucket_name (str): The name of the bucket to delete. - force (bool): If True, empty the bucket before deleting it. - - Returns: - None - """ - self.loggit.info(f"Deleting bucket: {bucket_name}") - try: - # If force=True, empty the bucket first - if force: - self.loggit.info(f"Emptying bucket {bucket_name} before deletion") - try: - # List and delete all objects - paginator = self.client.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name) - - for page in pages: - if "Contents" in page: - objects = [{"Key": obj["Key"]} for obj in page["Contents"]] - if objects: - self.client.delete_objects( - Bucket=bucket_name, Delete={"Objects": objects} - ) - self.loggit.debug( - f"Deleted {len(objects)} objects from {bucket_name}" - ) - except ClientError as e: - if e.response["Error"]["Code"] != "NoSuchBucket": - self.loggit.warning(f"Error emptying bucket {bucket_name}: {e}") - - self.client.delete_bucket(Bucket=bucket_name) - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def put_object(self, bucket_name: str, key: str, body: str = "") -> None: - """ - Put an object in a bucket. - - Args: - bucket_name (str): The name of the bucket to put the object in. - key (str): The key of the object to put. - body (str): The body of the object to put. - - Returns: - None - """ - self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") - try: - self.client.put_object(Bucket=bucket_name, Key=key, Body=body) - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def list_buckets(self, prefix: str = None) -> list[str]: - """ - List all buckets. - - Returns: - list[str]: A list of bucket names. - """ - self.loggit.info("Listing buckets") - try: - response = self.client.list_buckets() - buckets = response.get("Buckets", []) - bucket_names = [bucket["Name"] for bucket in buckets] - if prefix: - bucket_names = [ - name for name in bucket_names if name.startswith(prefix) - ] - return bucket_names - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - def head_object(self, bucket_name: str, key: str) -> dict: - """ - Retrieve metadata for an object without downloading it. - - Args: - bucket_name (str): The name of the bucket. - key (str): The object key. - - Returns: - dict: Object metadata including Restore status if applicable. - """ - self.loggit.debug(f"Getting metadata for s3://{bucket_name}/{key}") - try: - response = self.client.head_object(Bucket=bucket_name, Key=key) - return response - except ClientError as e: - self.loggit.error(f"Error getting metadata for {key}: {e}") - raise ActionError(f"Error getting metadata for {key}: {e}") from e - - def copy_object( - self, - Bucket: str, - Key: str, - CopySource: dict[str, str], - StorageClass: str = "GLACIER", - ) -> None: - """ - Copy an object from one bucket to another. - - Args: - Bucket (str): The name of the destination bucket. - Key (str): The key for the copied object. - CopySource (dict[str, str]): The source bucket and key. - StorageClass (str): The storage class to use. - - Returns: - None - """ - self.loggit.info(f"Copying object {Key} to bucket {Bucket}") - try: - self.client.copy_object( - Bucket=Bucket, - CopySource=CopySource, - Key=Key, - StorageClass=StorageClass, - ) - except ClientError as e: - self.loggit.error(e) - raise ActionError(e) from e - - -def s3_client_factory(provider: str) -> S3Client: +def s3_client_factory(provider: str, **kwargs) -> S3Client: """ s3_client_factory method, returns an S3Client object implemented specific to the value of the provider argument. Args: provider (str): The provider to use for the S3Client object. Should - reference an implemented provider (aws, gcp, azure, etc) + reference an implemented provider (aws, azure, gcp, etc) + **kwargs: Provider-specific configuration options: + + AWS options: + region (str): AWS region (e.g., 'us-east-1') + profile (str): AWS profile name from ~/.aws/credentials + access_key_id (str): AWS access key ID + secret_access_key (str): AWS secret access key + + Azure options: + connection_string (str): Azure Storage connection string + account_name (str): Azure Storage account name + account_key (str): Azure Storage account key + + GCP options: + project (str): GCP project ID + credentials_file (str): Path to service account JSON file + location (str): Default location for bucket creation Raises: NotImplementedError: raised if the provider is not implemented @@ -640,12 +214,22 @@ def s3_client_factory(provider: str) -> S3Client: S3Client: An S3Client object specific to the provider argument. """ if provider == "aws": - return AwsS3Client() - elif provider == "gcp": - # Placeholder for GCP S3Client implementation - raise NotImplementedError("GCP S3Client is not implemented yet") + from deepfreeze_core.aws_client import AwsS3Client + + return AwsS3Client(**kwargs) elif provider == "azure": - # Placeholder for Azure S3Client implementation - raise NotImplementedError("Azure S3Client is not implemented yet") + from deepfreeze_core.azure_client import AzureBlobClient + + return AzureBlobClient(**kwargs) + elif provider == "gcp": + from deepfreeze_core.gcp_client import GcpStorageClient + + return GcpStorageClient(**kwargs) else: raise ValueError(f"Unsupported provider: {provider}") + + +# Backward-compatible re-exports +from deepfreeze_core.aws_client import AwsS3Client # noqa: E402, F401 + +__all__ = ["S3Client", "AwsS3Client", "s3_client_factory"] diff --git a/packages/deepfreeze-core/deepfreeze_core/utilities.py b/packages/deepfreeze-core/deepfreeze_core/utilities.py index 2fc28cc..a192118 100644 --- a/packages/deepfreeze-core/deepfreeze_core/utilities.py +++ b/packages/deepfreeze-core/deepfreeze_core/utilities.py @@ -24,79 +24,58 @@ from deepfreeze_core.s3client import S3Client -def push_to_glacier(s3: S3Client, repo: Repository) -> bool: - """Push objects to Glacier storage +def push_to_glacier(s3: S3Client, repo: Repository, storage_class: str = "GLACIER") -> bool: + """Push objects to archive storage (Glacier for AWS, Archive tier for Azure, etc.) - :param s3: The S3 client object + Uses the storage client's refreeze method which handles provider-specific + archiving correctly: + - AWS S3: Copy-in-place with new storage class + - Azure: Direct tier change with set_standard_blob_tier() + - GCP: Direct storage class change + + :param s3: The storage client object :type s3: S3Client - :param repo: The repository to push to Glacier + :param repo: The repository to archive :type repo: Repository + :param storage_class: Target storage class (default: GLACIER) + :type storage_class: str - :return: True if all objects were successfully moved, False otherwise + :return: True if archiving completed (may have partial failures), False on error :rtype: bool - - :raises Exception: If the object is not in the restoration process """ + loggit = logging.getLogger("deepfreeze.utilities") + try: - # Normalize base_path: remove leading/trailing slashes, ensure it ends with / + # Normalize base_path: remove leading/trailing slashes base_path = repo.base_path.strip("/") if base_path: base_path += "/" - # Initialize variables for pagination - success = True - object_count = 0 - - # List objects - objects = s3.list_objects(repo.bucket, base_path) - - # Process each object - for obj in objects: - key = obj["Key"] - current_storage_class = obj.get("StorageClass", "STANDARD") - - # Log the object being processed - logging.info( - "Processing object: s3://%s/%s (Current: %s)", - repo.bucket, - key, - current_storage_class, - ) - - try: - # Copy object to itself with new storage class - copy_source = {"Bucket": repo.bucket, "Key": key} - s3.copy_object( - Bucket=repo.bucket, - Key=key, - CopySource=copy_source, - StorageClass="GLACIER", - ) - - # Log success - logging.info( - "Successfully moved s3://%s/%s to GLACIER", repo.bucket, key - ) - object_count += 1 + loggit.info( + "Archiving repository %s (bucket: %s, path: %s) to %s", + repo.name, + repo.bucket, + base_path, + storage_class, + ) - except botocore.exceptions.ClientError as e: - logging.error("Failed to move s3://%s/%s: %s", repo.bucket, key, e) - success = False - continue + # Use the storage client's refreeze method which handles provider-specific + # archiving correctly (direct tier change for Azure/GCP, copy-in-place for AWS) + s3.refreeze(repo.bucket, base_path, storage_class) - # Log summary - logging.info( - "Processed %d objects in s3://%s/%s", object_count, repo.bucket, base_path + loggit.info( + "Archive operation completed for repository %s", + repo.name, ) - if success: - logging.info("All objects successfully moved to GLACIER") - else: - logging.warning("Some objects failed to move to GLACIER") - - return success + return True - except botocore.exceptions.ClientError as e: - logging.error("Failed to process bucket s3://%s: %s", repo.bucket, e) + except Exception as e: + loggit.error( + "Failed to archive repository %s: %s", + repo.name, + e, + exc_info=True, + ) return False @@ -122,6 +101,71 @@ def get_all_indices_in_repo(client: Elasticsearch, repository: str) -> list: return list(indices) +def repo_has_active_indices(client: Elasticsearch, repo_name: str) -> tuple[bool, list]: + """ + Check if a repository has any active searchable snapshot indices using it. + + This is used to determine if a repository can be safely archived/unmounted. + Elasticsearch won't allow unmounting a repo that has active indices. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The repository name to check + :type repo_name: str + + :returns: Tuple of (has_active_indices, list_of_index_names) + :rtype: tuple[bool, list] + """ + loggit = logging.getLogger("deepfreeze.utilities") + active_indices = [] + + try: + # Get all indices and check which ones are searchable snapshots from this repo + all_indices = client.cat.indices(format="json") + + for idx in all_indices: + index_name = idx.get("index", "") + # Searchable snapshots have names like "partial---" + # or the index settings will reference the repository + if repo_name in index_name: + active_indices.append(index_name) + continue + + # Also check index settings for repository reference + try: + settings = client.indices.get_settings(index=index_name) + store_type = ( + settings.get(index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("store", {}) + .get("type") + ) + snapshot_repo = ( + settings.get(index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("store", {}) + .get("snapshot", {}) + .get("repository_name") + ) + if store_type == "snapshot" and snapshot_repo == repo_name: + active_indices.append(index_name) + except Exception: + pass # Index might not exist or be accessible + + except Exception as e: + loggit.warning("Error checking for active indices in repo %s: %s", repo_name, e) + + loggit.debug( + "Repository %s has %d active indices: %s", + repo_name, + len(active_indices), + active_indices[:5] if active_indices else [], + ) + return len(active_indices) > 0, active_indices + + def get_timestamp_range(client: Elasticsearch, indices: list) -> tuple: """ Retrieve the earliest and latest @timestamp values from the given indices. @@ -155,6 +199,10 @@ def get_timestamp_range(client: Elasticsearch, indices: list) -> tuple: filtered = [index for index in indices if client.indices.exists(index=index)] logging.debug("after removing non-existent indices: %s", len(filtered)) + if not filtered: + logging.debug("No existing indices remain after filtering") + return None, None + try: response = client.search( index=",".join(filtered), body=query, allow_partial_search_results=True @@ -164,12 +212,18 @@ def get_timestamp_range(client: Elasticsearch, indices: list) -> tuple: logging.error("Error retrieving timestamp range: %s", e) return None, None - earliest = response["aggregations"]["earliest"]["value_as_string"] - latest = response["aggregations"]["latest"]["value_as_string"] + earliest_val = response["aggregations"]["earliest"].get("value_as_string") + latest_val = response["aggregations"]["latest"].get("value_as_string") + + if not earliest_val or not latest_val: + logging.debug( + "Aggregation returned null timestamps (indices may have no @timestamp data)" + ) + return None, None - logging.debug("Earliest: %s, Latest: %s", earliest, latest) + logging.debug("Earliest: %s, Latest: %s", earliest_val, latest_val) - return datetime.fromisoformat(earliest), datetime.fromisoformat(latest) + return datetime.fromisoformat(earliest_val), datetime.fromisoformat(latest_val) def ensure_settings_index( @@ -317,43 +371,64 @@ def create_repo( base_path: str, canned_acl: str, storage_class: str, + provider: str = "aws", dry_run: bool = False, ) -> None: """ - Creates a new repo using the previously-created bucket. + Creates a new repo using the previously-created bucket/container. :param client: A client connection object :type client: Elasticsearch :param repo_name: The name of the repository to create :type repo_name: str - :param bucket_name: The name of the bucket to use for the repository + :param bucket_name: The name of the bucket/container to use for the repository :type bucket_name: str :param base_path: Path within a bucket where snapshots are stored :type base_path: str - :param canned_acl: One of the AWS canned ACL values + :param canned_acl: One of the AWS canned ACL values (AWS only) :type canned_acl: str - :param storage_class: AWS Storage class + :param storage_class: Storage class (AWS only) :type storage_class: str + :param provider: Cloud provider (aws, azure, gcp) + :type provider: str :param dry_run: If True, do not actually create the repository :type dry_run: bool :raises ActionError: If the repository cannot be created """ loggit = logging.getLogger("deepfreeze.utilities") - loggit.info("Creating repo %s using bucket %s", repo_name, bucket_name) + loggit.info("Creating repo %s using bucket %s (provider: %s)", repo_name, bucket_name, provider) if dry_run: return + + # Build repository settings based on provider + if provider == "azure": + repo_type = "azure" + settings = { + "container": bucket_name, + "base_path": base_path, + } + elif provider == "gcp": + repo_type = "gcs" + settings = { + "bucket": bucket_name, + "base_path": base_path, + } + else: # aws + repo_type = "s3" + settings = { + "bucket": bucket_name, + "base_path": base_path, + "canned_acl": canned_acl, + "storage_class": storage_class, + } + try: client.snapshot.create_repository( name=repo_name, body={ - "type": "s3", - "settings": { - "bucket": bucket_name, - "base_path": base_path, - "canned_acl": canned_acl, - "storage_class": storage_class, - }, + "type": repo_type, + "settings": settings, }, ) except Exception as e: @@ -545,7 +620,8 @@ def unmount_repo(client: Elasticsearch, repo: str) -> Repository: loggit = logging.getLogger("deepfreeze.utilities") # Get repository info from Elasticsearch repo_info = client.snapshot.get_repository(name=repo)[repo] - bucket = repo_info["settings"]["bucket"] + # Handle different providers: AWS/GCP use "bucket", Azure uses "container" + bucket = repo_info["settings"].get("bucket") or repo_info["settings"].get("container") base_path = repo_info["settings"]["base_path"] # Get repository object from status index @@ -565,21 +641,21 @@ def unmount_repo(client: Elasticsearch, repo: str) -> Repository: repo_obj.end.isoformat() if repo_obj.end else "None", ) - # Mark repository as unmounted - repo_obj.unmount() - msg = f"Recording repository details as {repo_obj}" - loggit.debug(msg) - # Remove the repository from Elasticsearch - loggit.debug("Removing repo %s", repo) + loggit.debug("Removing repo %s from Elasticsearch", repo) try: client.snapshot.delete_repository(name=repo) except Exception as e: loggit.warning("Repository %s could not be unmounted due to %s", repo, e) loggit.warning("Another attempt will be made when rotate runs next") + # Don't mark as unmounted if the actual unmount failed + raise + + # Only mark as unmounted AFTER successful ES unmount + repo_obj.unmount() + loggit.debug("Recording repository details as %s", repo_obj) # Update the status index with final repository state - loggit.debug("Updating repo: %s", repo_obj) client.update(index=STATUS_INDEX, doc=repo_obj.to_dict(), id=repo_obj.docid) loggit.debug("Repo %s removed", repo) return repo_obj @@ -815,92 +891,87 @@ def update_index_template_ilm_policy( # First try composable templates (ES 7.8+) try: templates = client.indices.get_index_template(name=template_name) - if ( - templates - and "index_templates" in templates - and len(templates["index_templates"]) > 0 - ): - template_data = templates["index_templates"][0]["index_template"] - loggit.debug("Found composable template %s", template_name) - - # Ensure template structure exists - if "template" not in template_data: - template_data["template"] = {} - if "settings" not in template_data["template"]: - template_data["template"]["settings"] = {} - if "index" not in template_data["template"]["settings"]: - template_data["template"]["settings"]["index"] = {} - if "lifecycle" not in template_data["template"]["settings"]["index"]: - template_data["template"]["settings"]["index"]["lifecycle"] = {} - - # Get old policy name for logging - old_policy = template_data["template"]["settings"]["index"][ - "lifecycle" - ].get("name", "none") - - # Set the new ILM policy - template_data["template"]["settings"]["index"]["lifecycle"][ - "name" - ] = ilm_policy_name - - # Put the updated template - client.indices.put_index_template(name=template_name, body=template_data) - loggit.info( - "Updated composable template %s: ILM policy %s -> %s", - template_name, - old_policy, - ilm_policy_name, - ) - return { - "action": "updated", - "template_type": "composable", - "old_policy": old_policy, - "new_policy": ilm_policy_name, - } except NotFoundError: + templates = None loggit.debug( "Composable template %s not found, trying legacy template", template_name ) except Exception as e: + templates = None loggit.debug("Error checking composable template %s: %s", template_name, e) + if ( + templates + and "index_templates" in templates + and len(templates["index_templates"]) > 0 + ): + template_data = templates["index_templates"][0]["index_template"] + loggit.debug("Found composable template %s", template_name) + + # Ensure template structure exists + if "template" not in template_data: + template_data["template"] = {} + if "settings" not in template_data["template"]: + template_data["template"]["settings"] = {} + if "index" not in template_data["template"]["settings"]: + template_data["template"]["settings"]["index"] = {} + if "lifecycle" not in template_data["template"]["settings"]["index"]: + template_data["template"]["settings"]["index"]["lifecycle"] = {} + + # Get old policy name for logging + old_policy = template_data["template"]["settings"]["index"][ + "lifecycle" + ].get("name", "none") + + # Set the new ILM policy + template_data["template"]["settings"]["index"]["lifecycle"][ + "name" + ] = ilm_policy_name + + # Only include fields accepted by put_index_template to avoid + # sending system-managed fields (e.g. created_date) from the GET response + _COMPOSABLE_TEMPLATE_FIELDS = { + "index_patterns", + "template", + "composed_of", + "priority", + "version", + "_meta", + "data_stream", + "allow_auto_create", + "deprecated", + "ignore_missing_component_templates", + } + put_body = {k: v for k, v in template_data.items() if k in _COMPOSABLE_TEMPLATE_FIELDS} + + # Put the updated template + try: + client.indices.put_index_template( + name=template_name, body=put_body + ) + except Exception as e: + loggit.error( + "Error updating composable template %s: %s", template_name, e + ) + raise ActionError( + f"Failed to update composable template {template_name}: {e}" + ) from e + loggit.info( + "Updated composable template %s: ILM policy %s -> %s", + template_name, + old_policy, + ilm_policy_name, + ) + return { + "action": "updated", + "template_type": "composable", + "old_policy": old_policy, + "new_policy": ilm_policy_name, + } + # Try legacy templates try: templates = client.indices.get_template(name=template_name) - if templates and template_name in templates: - template_data = templates[template_name] - loggit.debug("Found legacy template %s", template_name) - - # Ensure template structure exists - if "settings" not in template_data: - template_data["settings"] = {} - if "index" not in template_data["settings"]: - template_data["settings"]["index"] = {} - if "lifecycle" not in template_data["settings"]["index"]: - template_data["settings"]["index"]["lifecycle"] = {} - - # Get old policy name for logging - old_policy = template_data["settings"]["index"]["lifecycle"].get( - "name", "none" - ) - - # Set the new ILM policy - template_data["settings"]["index"]["lifecycle"]["name"] = ilm_policy_name - - # Put the updated template - client.indices.put_template(name=template_name, body=template_data) - loggit.info( - "Updated legacy template %s: ILM policy %s -> %s", - template_name, - old_policy, - ilm_policy_name, - ) - return { - "action": "updated", - "template_type": "legacy", - "old_policy": old_policy, - "new_policy": ilm_policy_name, - } except NotFoundError: loggit.warning( "Template %s not found (checked both composable and legacy)", template_name @@ -911,8 +982,49 @@ def update_index_template_ilm_policy( "error": f"Template {template_name} not found", } except Exception as e: - loggit.error("Error updating legacy template %s: %s", template_name, e) - raise ActionError(f"Failed to update template {template_name}: {e}") from e + loggit.error("Error checking legacy template %s: %s", template_name, e) + raise ActionError(f"Failed to check legacy template {template_name}: {e}") from e + + if templates and template_name in templates: + template_data = templates[template_name] + loggit.debug("Found legacy template %s", template_name) + + # Ensure template structure exists + if "settings" not in template_data: + template_data["settings"] = {} + if "index" not in template_data["settings"]: + template_data["settings"]["index"] = {} + if "lifecycle" not in template_data["settings"]["index"]: + template_data["settings"]["index"]["lifecycle"] = {} + + # Get old policy name for logging + old_policy = template_data["settings"]["index"]["lifecycle"].get( + "name", "none" + ) + + # Set the new ILM policy + template_data["settings"]["index"]["lifecycle"]["name"] = ilm_policy_name + + # Put the updated template + try: + client.indices.put_template(name=template_name, body=template_data) + except Exception as e: + loggit.error("Error updating legacy template %s: %s", template_name, e) + raise ActionError( + f"Failed to update legacy template {template_name}: {e}" + ) from e + loggit.info( + "Updated legacy template %s: ILM policy %s -> %s", + template_name, + old_policy, + ilm_policy_name, + ) + return { + "action": "updated", + "template_type": "legacy", + "old_policy": old_policy, + "new_policy": ilm_policy_name, + } def create_thawed_ilm_policy(client: Elasticsearch, repo_name: str) -> str: @@ -1018,6 +1130,19 @@ def update_repository_date_range(client: Elasticsearch, repo: Repository) -> boo # Find which indices are actually mounted (try multiple naming patterns) mounted_indices = [] for idx in snapshot_indices: + # ILM force-merge creates snapshots with fm-clone-xxxx- prefix, + # but mounted indices use the original name. Strip that prefix. + # Pattern: fm-clone-- -> + if idx.startswith("fm-clone-"): + # Find the second hyphen (after random chars) and strip prefix + parts = idx.split("-", 3) # ['fm', 'clone', 'random', 'rest'] + if len(parts) >= 4: + original_idx = parts[3] # Everything after fm-clone-xxxx- + loggit.debug( + "Stripped fm-clone prefix: %s -> %s", idx, original_idx + ) + idx = original_idx + # Try original name if client.indices.exists(index=idx): mounted_indices.append(idx) @@ -1301,21 +1426,46 @@ def mount_repo(client: Elasticsearch, repo: Repository) -> None: loggit = logging.getLogger("deepfreeze.utilities") loggit.info("Mounting repository %s", repo.name) - # Get settings to retrieve canned_acl and storage_class + # Get settings to retrieve provider and storage settings settings = get_settings(client) + provider = settings.provider + + # Build repository settings based on provider + if provider == "azure": + repo_type = "azure" + repo_settings = { + "container": repo.bucket, + "base_path": repo.base_path, + } + elif provider == "gcp": + repo_type = "gcs" + repo_settings = { + "bucket": repo.bucket, + "base_path": repo.base_path, + } + else: # aws (default) + repo_type = "s3" + repo_settings = { + "bucket": repo.bucket, + "base_path": repo.base_path, + "canned_acl": settings.canned_acl, + "storage_class": settings.storage_class, + } + + loggit.debug( + "Mounting repository %s with type=%s, settings=%s", + repo.name, + repo_type, + repo_settings, + ) # Create the repository in Elasticsearch try: client.snapshot.create_repository( name=repo.name, body={ - "type": "s3", - "settings": { - "bucket": repo.bucket, - "base_path": repo.base_path, - "canned_acl": settings.canned_acl, - "storage_class": settings.storage_class, - }, + "type": repo_type, + "settings": repo_settings, }, ) loggit.info("Repository %s created successfully", repo.name) @@ -1629,7 +1779,23 @@ def update_template_ilm_policy( "name" ] = new_policy_name - client.indices.put_index_template(name=template_name, body=template) + # Only include fields accepted by put_index_template to avoid + # sending system-managed fields (e.g. created_date) from the GET response + _COMPOSABLE_FIELDS = { + "index_patterns", + "template", + "composed_of", + "priority", + "version", + "_meta", + "data_stream", + "allow_auto_create", + "deprecated", + "ignore_missing_component_templates", + } + put_body = {k: v for k, v in template.items() if k in _COMPOSABLE_FIELDS} + + client.indices.put_index_template(name=template_name, body=put_body) loggit.info( "Updated composable template %s to use policy %s", template_name, diff --git a/packages/deepfreeze-core/pyproject.toml b/packages/deepfreeze-core/pyproject.toml index 687de5c..53d6eb8 100644 --- a/packages/deepfreeze-core/pyproject.toml +++ b/packages/deepfreeze-core/pyproject.toml @@ -42,6 +42,12 @@ dependencies = [ ] [project.optional-dependencies] +azure = [ + "azure-storage-blob>=12.0.0", +] +gcp = [ + "google-cloud-storage>=2.0.0", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", @@ -49,6 +55,8 @@ dev = [ "black>=23.0.0", "ruff>=0.1.0", "moto>=4.0.0", + "azure-storage-blob>=12.0.0", + "google-cloud-storage>=2.0.0", ] [project.urls] diff --git a/tests/cli/test_actions.py b/tests/cli/test_actions.py index 281c46f..1245973 100644 --- a/tests/cli/test_actions.py +++ b/tests/cli/test_actions.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch import pytest +from rich.console import Console from deepfreeze import ( Cleanup, MissingIndexError, @@ -282,6 +283,53 @@ def test_rotate_calculates_next_suffix(self): assert new_repo == "deepfreeze-000006" + def test_update_date_ranges_calls_for_each_mounted_repo(self): + """Test _update_date_ranges calls update_repository_date_range for each mounted repo""" + mock_client = MagicMock() + mock_client.indices.exists.return_value = True + + mock_settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + style="oneup", + last_suffix="000001", + ) + + repo1 = Repository(name="deepfreeze-000001", bucket="bucket1", is_mounted=True) + repo2 = Repository(name="deepfreeze-000002", bucket="bucket2", is_mounted=True) + + with patch("deepfreeze_core.actions.rotate.get_settings") as mock_get: + mock_get.return_value = mock_settings + + with patch( + "deepfreeze_core.actions.rotate.s3_client_factory" + ) as mock_factory: + mock_s3 = MagicMock() + mock_factory.return_value = mock_s3 + + with patch( + "deepfreeze_core.actions.rotate.get_matching_repos" + ) as mock_repos: + mock_repos.return_value = [repo1, repo2] + + with patch( + "deepfreeze_core.actions.rotate.update_repository_date_range" + ) as mock_update: + # First repo updated, second repo unchanged + mock_update.side_effect = [True, False] + + rotate = Rotate(client=mock_client, porcelain=True) + rotate._load_settings() + + updated = rotate._update_date_ranges() + + assert mock_update.call_count == 2 + mock_update.assert_any_call(mock_client, repo1) + mock_update.assert_any_call(mock_client, repo2) + assert updated == ["deepfreeze-000001"] + + class TestThawAction: """Tests for the Thaw action class""" @@ -575,7 +623,8 @@ def test_repair_metadata_dry_run_scans_repos(self): repair = RepairMetadata(client=mock_client, porcelain=True) repair.do_dry_run() - mock_repos.assert_called_once() + # Called twice: once for state scan, once for date range update + assert mock_repos.call_count == 2 class TestActionInterfaceConsistency: @@ -678,6 +727,93 @@ def test_setup_with_canned_acl(self): class TestStatusActionAdditional: """Additional tests for Status action (Task Group 18)""" + def test_status_rich_output_date_only(self): + """Test Status rich output truncates dates when show_time is False""" + status = Status( + client=MagicMock(), + porcelain=False, + show_repos=True, + show_thawed=True, + show_buckets=False, + show_ilm=False, + show_config=False, + ) + status.console = Console(stderr=True, record=True, width=200) + + repos = [ + { + "name": "repo-1", + "bucket": "bucket-1", + "base_path": "path", + "start": "2024-01-01T00:00:00+00:00", + "end": "2024-01-31T23:59:59+00:00", + "is_mounted": True, + "thaw_state": "frozen", + "storage_tier": "Archive", + } + ] + thaw_requests = [ + { + "request_id": "req-1", + "status": "completed", + "start_date": "2024-01-01T00:00:00+00:00", + "end_date": "2024-01-31T23:59:59+00:00", + "repos": ["repo-1"], + "created_at": "2024-02-01T12:34:56+00:00", + } + ] + + status._display_rich(repos, thaw_requests, buckets=[], ilm_policies=[]) + + output = status.console.export_text() + assert "2024-01-01 - 2024-01-31" in output + assert "2024-02-01 12:34" in output + assert "2024-01-01 00:00:00+00:00" not in output + assert "2024-02-01 12:34:56+00:00" not in output + + def test_status_rich_output_with_time(self): + """Test Status rich output shows full datetime when show_time is True""" + status = Status( + client=MagicMock(), + porcelain=False, + show_repos=True, + show_thawed=True, + show_buckets=False, + show_ilm=False, + show_config=False, + show_time=True, + ) + status.console = Console(stderr=True, record=True, width=200) + + repos = [ + { + "name": "repo-1", + "bucket": "bucket-1", + "base_path": "path", + "start": "2024-01-01T00:00:00+00:00", + "end": "2024-01-31T23:59:59+00:00", + "is_mounted": True, + "thaw_state": "frozen", + "storage_tier": "Archive", + } + ] + thaw_requests = [ + { + "request_id": "req-1", + "status": "completed", + "start_date": "2024-01-01T00:00:00+00:00", + "end_date": "2024-01-31T23:59:59+00:00", + "repos": ["repo-1"], + "created_at": "2024-02-01T12:34:56+00:00", + } + ] + + status._display_rich(repos, thaw_requests, buckets=[], ilm_policies=[]) + + output = status.console.export_text() + assert "2024-01-01 00:00:00+00:00 - 2024-01-31 23:59:59+00:00" in output + assert "2024-02-01 12:34:56+00:00" in output + def test_status_porcelain_mode(self): """Test Status with porcelain=True produces machine-readable output""" mock_client = MagicMock() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 59ee38b..3d8d839 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -197,6 +197,7 @@ def test_status_command_options(self, runner): assert "--limit" in result.output assert "--repos" in result.output assert "--porcelain" in result.output + assert "--time" in result.output def test_rotate_command_options(self, runner): """Test that rotate command has expected options.""" diff --git a/tests/cli/test_exceptions_constants.py b/tests/cli/test_exceptions_constants.py index 52e1522..56427d8 100644 --- a/tests/cli/test_exceptions_constants.py +++ b/tests/cli/test_exceptions_constants.py @@ -80,7 +80,7 @@ def test_constants_importable(): # Verify values assert STATUS_INDEX == "deepfreeze-status" assert SETTINGS_ID == "1" - assert PROVIDERS == ["aws"] + assert PROVIDERS == ["aws", "azure", "gcp"] # Verify thaw states assert THAW_STATE_ACTIVE == "active" diff --git a/tests/cli/test_integration.py b/tests/cli/test_integration.py index b39ff83..496ffbb 100644 --- a/tests/cli/test_integration.py +++ b/tests/cli/test_integration.py @@ -112,6 +112,15 @@ def _search_documents(self, index=None, body=None, **kwargs): class MockS3Client: """Mock S3 client for integration testing""" + # Elasticsearch repository plugin information (matches AwsS3Client defaults) + ES_PLUGIN_NAME = "repository-s3" + ES_PLUGIN_DISPLAY_NAME = "S3" + ES_PLUGIN_DOC_URL = "https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" + STORAGE_TYPE = "S3 bucket" + ES_KEYSTORE_INSTRUCTIONS = "Configure AWS credentials in Elasticsearch keystore" + STORAGE_CREATION_HELP = "Check AWS credentials and permissions" + STORAGE_DELETE_CMD = "aws s3 rb s3://{bucket} --force" + def __init__(self): self._buckets = {} self._objects = {} diff --git a/tests/cli/test_s3client.py b/tests/cli/test_s3client.py index eefaea5..3b3be9d 100644 --- a/tests/cli/test_s3client.py +++ b/tests/cli/test_s3client.py @@ -24,7 +24,7 @@ class TestAwsS3ClientInstantiation: """Tests for AwsS3Client instantiation""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_client_instantiation_success(self, mock_boto3): """Test that AwsS3Client can be instantiated with valid credentials""" from deepfreeze import AwsS3Client @@ -41,7 +41,7 @@ def test_client_instantiation_success(self, mock_boto3): mock_boto3.client.assert_called_once_with("s3") mock_client.list_buckets.assert_called_once() - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_client_instantiation_invalid_credentials(self, mock_boto3): """Test that AwsS3Client raises ActionError for invalid credentials""" from botocore.exceptions import ClientError @@ -66,7 +66,7 @@ def test_client_instantiation_invalid_credentials(self, mock_boto3): class TestBucketExists: """Tests for bucket_exists method""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_bucket_exists_true(self, mock_boto3): """Test bucket_exists returns True when bucket exists""" from deepfreeze import AwsS3Client @@ -82,7 +82,7 @@ def test_bucket_exists_true(self, mock_boto3): assert result is True mock_client.head_bucket.assert_called_once_with(Bucket="test-bucket") - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_bucket_exists_false(self, mock_boto3): """Test bucket_exists returns False when bucket does not exist""" from botocore.exceptions import ClientError @@ -103,7 +103,7 @@ def test_bucket_exists_false(self, mock_boto3): class TestListObjects: """Tests for list_objects method""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_list_objects_empty(self, mock_boto3): """Test list_objects returns empty list when no objects""" from deepfreeze import AwsS3Client @@ -122,7 +122,7 @@ def test_list_objects_empty(self, mock_boto3): assert result == [] - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_list_objects_with_results(self, mock_boto3): """Test list_objects returns objects when they exist""" from deepfreeze import AwsS3Client @@ -151,7 +151,7 @@ def test_list_objects_with_results(self, mock_boto3): class TestS3ClientFactory: """Tests for s3_client_factory function""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_factory_returns_aws_client(self, mock_boto3): """Test factory returns AwsS3Client for 'aws' provider""" from deepfreeze import AwsS3Client, s3_client_factory @@ -164,23 +164,31 @@ def test_factory_returns_aws_client(self, mock_boto3): assert isinstance(client, AwsS3Client) - def test_factory_raises_not_implemented_for_gcp(self): - """Test factory raises NotImplementedError for 'gcp' provider""" + @patch("deepfreeze_core.gcp_client.storage") + def test_factory_returns_gcp_client(self, mock_storage): + """Test factory returns GcpStorageClient for 'gcp' provider""" from deepfreeze import s3_client_factory + from deepfreeze_core import GcpStorageClient - with pytest.raises(NotImplementedError) as exc_info: - s3_client_factory("gcp") + mock_gcp_client = MagicMock() + mock_storage.Client.return_value = mock_gcp_client - assert "GCP S3Client is not implemented" in str(exc_info.value) + client = s3_client_factory("gcp") - def test_factory_raises_not_implemented_for_azure(self): - """Test factory raises NotImplementedError for 'azure' provider""" + assert isinstance(client, GcpStorageClient) + + @patch("deepfreeze_core.azure_client.BlobServiceClient") + def test_factory_returns_azure_client(self, mock_blob_service): + """Test factory returns AzureBlobClient for 'azure' provider""" from deepfreeze import s3_client_factory + from deepfreeze_core import AzureBlobClient + + mock_azure_client = MagicMock() + mock_blob_service.from_connection_string.return_value = mock_azure_client - with pytest.raises(NotImplementedError) as exc_info: - s3_client_factory("azure") + client = s3_client_factory("azure", connection_string="test-connection-string") - assert "Azure S3Client is not implemented" in str(exc_info.value) + assert isinstance(client, AzureBlobClient) def test_factory_raises_value_error_for_invalid_provider(self): """Test factory raises ValueError for invalid provider""" @@ -195,7 +203,7 @@ def test_factory_raises_value_error_for_invalid_provider(self): class TestS3ClientOperations: """Tests for various S3Client operations""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_test_connection_success(self, mock_boto3): """Test test_connection returns True when connection succeeds""" from deepfreeze import AwsS3Client @@ -209,7 +217,7 @@ def test_test_connection_success(self, mock_boto3): assert result is True - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_test_connection_failure(self, mock_boto3): """Test test_connection returns False when connection fails""" from botocore.exceptions import ClientError @@ -237,7 +245,7 @@ def test_test_connection_failure(self, mock_boto3): class TestBucketOperations: """Tests for bucket create and delete operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_create_bucket_success(self, mock_boto3): """Test create_bucket creates bucket successfully""" from deepfreeze import AwsS3Client @@ -262,7 +270,7 @@ def test_create_bucket_success(self, mock_boto3): mock_client.create_bucket.assert_called_once_with(Bucket="new-bucket") - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_create_bucket_already_exists(self, mock_boto3): """Test create_bucket raises error when bucket exists""" from deepfreeze import ActionError, AwsS3Client @@ -279,7 +287,7 @@ def test_create_bucket_already_exists(self, mock_boto3): assert "already exists" in str(exc_info.value) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_delete_bucket_success(self, mock_boto3): """Test delete_bucket deletes bucket successfully""" from deepfreeze import AwsS3Client @@ -293,7 +301,7 @@ def test_delete_bucket_success(self, mock_boto3): mock_client.delete_bucket.assert_called_once_with(Bucket="delete-me-bucket") - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_delete_bucket_force(self, mock_boto3): """Test delete_bucket with force=True empties bucket first""" from deepfreeze import AwsS3Client @@ -318,7 +326,7 @@ def test_delete_bucket_force(self, mock_boto3): class TestObjectOperations: """Tests for object put, head, and copy operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_put_object_success(self, mock_boto3): """Test put_object successfully puts object""" from deepfreeze import AwsS3Client @@ -334,7 +342,7 @@ def test_put_object_success(self, mock_boto3): Bucket="test-bucket", Key="test-key", Body="test-body" ) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_head_object_success(self, mock_boto3): """Test head_object returns metadata""" from deepfreeze import AwsS3Client @@ -357,7 +365,7 @@ def test_head_object_success(self, mock_boto3): Bucket="test-bucket", Key="test-key" ) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_copy_object_success(self, mock_boto3): """Test copy_object copies with storage class""" from deepfreeze import AwsS3Client @@ -381,7 +389,7 @@ def test_copy_object_success(self, mock_boto3): StorageClass="GLACIER", ) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_list_buckets_with_prefix(self, mock_boto3): """Test list_buckets filters by prefix""" from deepfreeze import AwsS3Client @@ -408,7 +416,7 @@ def test_list_buckets_with_prefix(self, mock_boto3): class TestThawRefreezeOperations: """Tests for thaw and refreeze operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_thaw_initiates_restore_for_glacier_objects(self, mock_boto3): """Test thaw initiates restore for Glacier storage class objects""" from deepfreeze import AwsS3Client @@ -436,7 +444,7 @@ def test_thaw_initiates_restore_for_glacier_objects(self, mock_boto3): # Should have called restore_object twice (only for GLACIER and DEEP_ARCHIVE) assert mock_client.restore_object.call_count == 2 - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_thaw_skips_objects_outside_base_path(self, mock_boto3): """Test thaw skips objects that don't start with base_path""" from deepfreeze import AwsS3Client @@ -463,7 +471,7 @@ def test_thaw_skips_objects_outside_base_path(self, mock_boto3): # Should have called restore_object only once assert mock_client.restore_object.call_count == 1 - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_refreeze_copies_objects_to_glacier(self, mock_boto3): """Test refreeze copies objects to GLACIER storage class""" from deepfreeze import AwsS3Client @@ -493,7 +501,7 @@ def test_refreeze_copies_objects_to_glacier(self, mock_boto3): class TestS3ClientErrorHandling: """Tests for error handling in S3 operations (Task Group 17)""" - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_head_object_raises_action_error(self, mock_boto3): """Test head_object raises ActionError on failure""" from botocore.exceptions import ClientError @@ -513,7 +521,7 @@ def test_head_object_raises_action_error(self, mock_boto3): assert "Error getting metadata" in str(exc_info.value) - @patch("deepfreeze_core.s3client.boto3") + @patch("deepfreeze_core.aws_client.boto3") def test_bucket_exists_raises_on_unexpected_error(self, mock_boto3): """Test bucket_exists raises ActionError on unexpected errors""" from botocore.exceptions import ClientError diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..f522afa --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.8" + +[[package]] +name = "deepfreeze-workspace" +version = "1.0.0" +source = { editable = "." }