diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d5e7b2ce07..b6877962ab 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -149,7 +149,7 @@ Options: ``` Usage: sqlmesh destroy - Removes all project resources, including warehouse objects, state tables, the SQLMesh cache and any build artifacts. + Removes all state tables, the SQLMesh cache and all project resources, including warehouse objects. This includes all tables, views and schemas managed by SQLMesh, as well as any external resources that may have been created by other tools within those schemas. Options: --help Show this message and exit. diff --git a/docs/reference/notebook.md b/docs/reference/notebook.md index 47c8731130..6cac4e1078 100644 --- a/docs/reference/notebook.md +++ b/docs/reference/notebook.md @@ -250,7 +250,7 @@ options: ``` %destroy -Removes all project resources, including warehouse objects, state tables, the SQLMesh cache and any build artifacts. +Removes all state tables, the SQLMesh cache, and other project resources, including warehouse objects. This includes all tables, views, and schemas managed by SQLMesh, as well as any external resources that may have been created by other tools within those schemas. ``` #### dlt_refresh diff --git a/sqlmesh/core/console.py b/sqlmesh/core/console.py index c14b9be2b5..118abcd769 100644 --- a/sqlmesh/core/console.py +++ b/sqlmesh/core/console.py @@ -186,9 +186,19 @@ class DestroyConsole(abc.ABC): """Console for describing a destroy operation""" @abc.abstractmethod - def start_destroy(self) -> bool: + def start_destroy( + self, + schemas_to_delete: t.Optional[t.Set[str]] = None, + views_to_delete: t.Optional[t.Set[str]] = None, + tables_to_delete: t.Optional[t.Set[str]] = None, + ) -> bool: """Start a destroy operation. + Args: + schemas_to_delete: Set of schemas that will be deleted + views_to_delete: Set of views that will be deleted + tables_to_delete: Set of tables that will be deleted + Returns: Whether or not the destroy operation should proceed """ @@ -830,7 +840,12 @@ def print_connection_config( ) -> None: pass - def start_destroy(self) -> bool: + def start_destroy( + self, + schemas_to_delete: t.Optional[t.Set[str]] = None, + views_to_delete: t.Optional[t.Set[str]] = None, + tables_to_delete: t.Optional[t.Set[str]] = None, + ) -> bool: return True def stop_destroy(self, success: bool = True) -> None: @@ -1282,16 +1297,40 @@ def stop_cleanup(self, success: bool = False) -> None: else: self.log_error("Cleanup failed!") - def start_destroy(self) -> bool: + def start_destroy( + self, + schemas_to_delete: t.Optional[t.Set[str]] = None, + views_to_delete: t.Optional[t.Set[str]] = None, + tables_to_delete: t.Optional[t.Set[str]] = None, + ) -> bool: self.log_warning( - ( - "This will permanently delete all engine-managed objects, state tables and SQLMesh cache.\n" - "The operation is irreversible and may disrupt any currently running or scheduled plans.\n" - "Use this command only when you intend to fully reset the project." - ) + "This will permanently delete all engine-managed objects, state tables and SQLMesh cache.\n" + "The operation may disrupt any currently running or scheduled plans.\n" ) - if not self._confirm("Proceed?"): - self.log_error("Destroy aborted!") + + if schemas_to_delete or views_to_delete or tables_to_delete: + if schemas_to_delete: + self.log_error("Schemas to be deleted:") + for schema in sorted(schemas_to_delete): + self.log_error(f" • {schema}") + + if views_to_delete: + self.log_error("\nEnvironment views to be deleted:") + for view in sorted(views_to_delete): + self.log_error(f" • {view}") + + if tables_to_delete: + self.log_error("\nSnapshot tables to be deleted:") + for table in sorted(tables_to_delete): + self.log_error(f" • {table}") + + self.log_error( + "\nThis action will DELETE ALL the above resources managed by SQLMesh AND\n" + "potentially external resources created by other tools in these schemas.\n" + ) + + if not self._confirm("Are you ABSOLUTELY SURE you want to proceed with deletion?"): + self.log_error("Destroy operation cancelled.") return False return True diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index d6a305f7c0..a2662ad0c2 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -858,10 +858,52 @@ def run_janitor(self, ignore_ttl: bool) -> bool: def destroy(self) -> bool: success = False - if self.console.start_destroy(): + # Collect resources to be deleted + environments = self.state_reader.get_environments() + schemas_to_delete = set() + tables_to_delete = set() + views_to_delete = set() + all_snapshot_infos = set() + + # For each environment find schemas and tables + for environment in environments: + all_snapshot_infos.update(environment.snapshots) + snapshots = self.state_reader.get_snapshots(environment.snapshots).values() + for snapshot in snapshots: + if snapshot.is_model and not snapshot.is_symbolic: + # Get the appropriate adapter + if environment.gateway_managed and snapshot.model_gateway: + adapter = self.engine_adapters.get( + snapshot.model_gateway, self.engine_adapter + ) + else: + adapter = self.engine_adapter + + if environment.suffix_target.is_schema or environment.suffix_target.is_catalog: + schema = snapshot.qualified_view_name.schema_for_environment( + environment.naming_info, dialect=adapter.dialect + ) + catalog = snapshot.qualified_view_name.catalog_for_environment( + environment.naming_info, dialect=adapter.dialect + ) + if catalog: + schemas_to_delete.add(f"{catalog}.{schema}") + else: + schemas_to_delete.add(schema) + + if environment.suffix_target.is_table: + view_name = snapshot.qualified_view_name.for_environment( + environment.naming_info, dialect=adapter.dialect + ) + views_to_delete.add(view_name) + + # Add snapshot tables + table_name = snapshot.table_name() + tables_to_delete.add(table_name) + + if self.console.start_destroy(schemas_to_delete, views_to_delete, tables_to_delete): try: - self._destroy() - success = True + success = self._destroy() finally: self.console.stop_destroy(success=success) @@ -2723,7 +2765,7 @@ def _context_diff( always_recreate_environment=always_recreate_environment, ) - def _destroy(self) -> None: + def _destroy(self) -> bool: # Invalidate all environments, including prod for environment in self.state_reader.get_environments(): self.state_sync.invalidate_environment(name=environment.name, protect_prod=False) @@ -2739,6 +2781,8 @@ def _destroy(self) -> None: # Finally clear caches self.clear_caches() + return True + def _run_janitor(self, ignore_ttl: bool = False) -> None: current_ts = now_timestamp() diff --git a/tests/core/test_integration.py b/tests/core/test_integration.py index d03db7af91..7337f8d3f4 100644 --- a/tests/core/test_integration.py +++ b/tests/core/test_integration.py @@ -6354,7 +6354,9 @@ def test_destroy(copy_to_temp_path): context.fetchdf(f"SELECT * FROM db_1.first_schema.model_two") # Use the destroy command to remove all data objects and state - context._destroy() + # Mock the console confirmation to automatically return True + with patch.object(context.console, "_confirm", return_value=True): + context._destroy() # Ensure all tables have been removed for table_name in state_tables: diff --git a/tests/integrations/jupyter/test_magics.py b/tests/integrations/jupyter/test_magics.py index 6507f07120..c8e38f448c 100644 --- a/tests/integrations/jupyter/test_magics.py +++ b/tests/integrations/jupyter/test_magics.py @@ -898,19 +898,19 @@ def test_destroy( assert not output.stderr text_output = convert_all_html_output_to_text(output) expected_messages = [ - "[WARNING] This will permanently delete all engine-managed objects, state tables and SQLMesh cache.\n" - "The operation is irreversible and may disrupt any currently running or scheduled plans.\n" - "Use this command only when you intend to fully reset the project.", + "[WARNING] This will permanently delete all engine-managed objects, state tables and SQLMesh cache.\nThe operation may disrupt any currently running or scheduled plans.", + "Schemas to be deleted:", + "• memory.sushi", + "Snapshot tables to be deleted:", + "This action will DELETE ALL the above resources managed by SQLMesh AND\npotentially external resources created by other tools in these schemas.", + "Are you ABSOLUTELY SURE you want to proceed with deletion? [y/n]:", "Environment 'prod' invalidated.", "Deleted object memory.sushi", 'Deleted object "memory"."raw"."model1"', - 'Deleted object "memory"."raw"."model1"', - 'Deleted object "memory"."raw"."model2"', 'Deleted object "memory"."raw"."model2"', 'Deleted object "memory"."raw"."demographics"', - 'Deleted object "memory"."raw"."demographics"', "State tables removed.", "Destroy completed successfully.", ] for message in expected_messages: - assert message in text_output + assert any(message in line for line in text_output)