Persist Content in Database

migrations/20250829081421_extend_contents_table.down.sql

@@ -0,0 +1,16 @@
+-- Add down migration script here
+-- Reverse the contents table extension
+
+-- Drop the indexes we created
+DROP INDEX IF EXISTS contents_clean_text_gin;
+DROP INDEX IF EXISTS contents_item_id_checksum_uq;
+
+-- Remove the new columns
+ALTER TABLE contents
+  DROP COLUMN clean_html,
+  DROP COLUMN clean_text;
+
+-- Restore original column names
+ALTER TABLE contents
+  RENAME COLUMN raw_html TO html,
+  RENAME COLUMN raw_text TO text;

migrations/20250829081421_extend_contents_table.up.sql

@@ -0,0 +1,16 @@
+-- Add up migration script here
+-- Extend contents table to support cleaned content persistence
+
+-- Add columns for cleaned versions while preserving original data
+ALTER TABLE contents RENAME COLUMN html TO raw_html;
+ALTER TABLE contents RENAME COLUMN text TO raw_text;
+
+ALTER TABLE contents ADD COLUMN clean_html TEXT;
+ALTER TABLE contents ADD COLUMN clean_text TEXT;
+
+-- Create composite unique index to prevent duplicate writes when content hasn't changed
+-- Note: checksum column already exists from original schema
+CREATE UNIQUE INDEX contents_item_id_checksum_uq ON contents(item_id, checksum) WHERE checksum IS NOT NULL;
+
+-- Optional: Add GIN index on clean_text for future full-text search capabilities
+CREATE INDEX contents_clean_text_gin ON contents USING GIN (to_tsvector('simple', clean_text)) WHERE clean_text IS NOT NULL;

src/entities/mod.rs

@@ -49,8 +49,10 @@ pub struct Item {
 #[derive(Debug, Clone, FromRow)]
 pub struct Content {
     pub item_id: Uuid, // PK and FK -> items.id
-    pub html: Option<String>,
-    pub text: Option<String>,
+    pub raw_html: Option<String>,
+    pub raw_text: Option<String>,
+    pub clean_html: Option<String>,
+    pub clean_text: Option<String>,
     pub lang: Option<String>,
     pub extracted_at: Option<DateTime<Utc>>,
     pub checksum: Option<String>,

src/jobs/handlers/fetch_page.rs

@@ -61,11 +61,11 @@ impl JobHandler for FetchPageJobHandler {
                 // Insert the content
                 sqlx::query!(
                     r#"
-                    INSERT INTO contents (item_id, html, text, lang, extracted_at, checksum)
+                    INSERT INTO contents (item_id, raw_html, raw_text, lang, extracted_at, checksum)
                     VALUES ($1, $2, NULL, NULL, NOW(), $3)
                     ON CONFLICT (item_id) 
                     DO UPDATE SET 
-                        html = EXCLUDED.html,
+                        raw_html = EXCLUDED.raw_html,
                         extracted_at = EXCLUDED.extracted_at,
                         checksum = EXCLUDED.checksum
                     "#,