Hello world
"# + .to_string(), + }; + + let base_url = Url::parse("https://example.com").unwrap(); + sanitize_and_resolve_links(&mut result, &base_url); + + assert!(!result.html.contains("
"#
+ .to_string(),
+ };
+
+ let base_url = Url::parse("https://example.com/article/").unwrap();
+ sanitize_and_resolve_links(&mut result, &base_url);
+
+ assert!(result.html.contains("https://example.com/page"));
+ assert!(
+ result
+ .html
+ .contains("https://example.com/article/image.jpg")
+ );
+ }
+
+ #[test]
+ fn test_normalize_whitespace() {
+ let text = " Hello world \n\n\n Test ";
+ let normalized = normalize_whitespace(text);
+ // The function preserves newlines and normalizes spaces
+ assert_eq!(normalized, "Hello world \n\n Test");
+ }
+}
diff --git a/src/extractor/language.rs b/src/extractor/language.rs
new file mode 100644
index 0000000..699ba60
--- /dev/null
+++ b/src/extractor/language.rs
@@ -0,0 +1,81 @@
+use whatlang::{Lang, detect};
+
+const MIN_CONFIDENCE: f64 = 0.25;
+const MIN_TEXT_LENGTH: usize = 50;
+
+pub fn detect_language(text: &str) -> OptionThis is the first paragraph of a sample article. It contains meaningful content that should be extracted and indexed for search purposes.
+This is the second paragraph with more substantial content. It includes a relative link and an image.
+
+ The article continues with more paragraphs containing valuable information for readers.
+Building better software requires attention to detail, proper testing, and good documentation. In this article, we'll explore the key principles.
+Each of these principles contributes to maintainable and reliable software systems that can evolve over time.
++ "Good code is its own best documentation." - Steve McConnell ++
Loading...
+ + diff --git a/src/extractor/tests/mod.rs b/src/extractor/tests/mod.rs new file mode 100644 index 0000000..9d3fa4a --- /dev/null +++ b/src/extractor/tests/mod.rs @@ -0,0 +1,144 @@ +use bytes::Bytes; +use chrono::Utc; +use reqwest::StatusCode; +use reqwest::header::HeaderMap; +use std::fs; +use url::Url; + +use crate::extractor::extract; +use crate::fetcher::types::{Charset, PageResponse}; + +#[tokio::test] +async fn test_extract_article() { + let html = fs::read_to_string("src/extractor/tests/fixtures/article.html") + .expect("Failed to read test fixture"); + + let response = create_test_response(html, "https://example.com/article"); + let result = extract(&response).await; + + assert!(result.is_some()); + let content = result.unwrap(); + + // The title might include the site name depending on the HTML structure + assert!(content.title.contains("Sample Article")); + assert_eq!(content.site_name, Some("News Site".to_string())); + assert!(content.text.contains("first paragraph")); + assert!(content.text.contains("second paragraph")); + assert!(!content.html.contains("{}
Unclosed tags