diff --git a/go/config-test-go-no-filter.json b/go/config-test-go-no-filter.json deleted file mode 100644 index 4f87d75..0000000 --- a/go/config-test-go-no-filter.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "couchDb": { - "url": "http://localhost:5984", - "user": "admin", - "password": "password" - }, - "mailSources": [ - { - "name": "Go Wildcard All Folders Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser1", - "password": "password123", - "mode": "archive", - "folderFilter": { - "include": ["*"], - "exclude": ["Drafts", "Trash"] - }, - "messageFilter": {} - }, - { - "name": "Go Work Pattern Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "syncuser", - "password": "syncpass", - "mode": "sync", - "folderFilter": { - "include": ["Work*", "Important*", "INBOX"], - "exclude": ["*Temp*"] - }, - "messageFilter": {} - }, - { - "name": "Go Specific Folders Only", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "archiveuser", - "password": "archivepass", - "mode": "archive", - "folderFilter": { - "include": ["INBOX", "Sent", "Personal"], - "exclude": [] - }, - "messageFilter": {} - } - ] -} \ No newline at end of file diff --git a/go/config-test-go.json b/go/config-test-go.json deleted file mode 100644 index ab20af5..0000000 --- a/go/config-test-go.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "couchDb": { - "url": "http://localhost:5984", - "user": "admin", - "password": "password" - }, - "mailSources": [ - { - "name": "Go Wildcard All Folders Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser1", - "password": "password123", - "mode": "archive", - "folderFilter": { - "include": ["*"], - "exclude": ["Drafts", "Trash"] - }, - "messageFilter": { - "subjectKeywords": ["meeting", "important"], - "senderKeywords": ["@company.com"] - } - }, - { - "name": "Go Work Pattern Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "syncuser", - "password": "syncpass", - "mode": "sync", - "folderFilter": { - "include": ["Work*", "Important*", "INBOX"], - "exclude": ["*Temp*"] - }, - "messageFilter": { - "recipientKeywords": ["support@", "team@"] - } - }, - { - "name": "Go Specific Folders Only", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "archiveuser", - "password": "archivepass", - "mode": "archive", - "folderFilter": { - "include": ["INBOX", "Sent", "Personal"], - "exclude": [] - }, - "messageFilter": {} - }, - { - "name": "Go Subfolder Pattern Test", - "enabled": false, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser2", - "password": "password456", - "mode": "archive", - "folderFilter": { - "include": ["Work/*", "Archive/*"], - "exclude": ["*/Drafts"] - }, - "messageFilter": {} - } - ] -} \ No newline at end of file diff --git a/go/mail/imap.go b/go/mail/imap.go index 712a0eb..6ba4453 100644 --- a/go/mail/imap.go +++ b/go/mail/imap.go @@ -6,6 +6,7 @@ import ( "io" "log" "mime" + "path/filepath" "strings" "time" @@ -82,84 +83,6 @@ func (c *ImapClient) ListMailboxes() ([]string, error) { return mailboxes, nil } -// ListFilteredMailboxes lists mailboxes matching the given folder filters using IMAP LIST -func (c *ImapClient) ListFilteredMailboxes(filter *config.FolderFilter) ([]string, error) { - var allMailboxes []string - - // If no include patterns, get all mailboxes - if len(filter.Include) == 0 { - return c.ListMailboxes() - } - - // Use IMAP LIST with each include pattern to let the server filter - seen := make(map[string]bool) - for _, pattern := range filter.Include { - cmd := c.List("", pattern, nil) - infos, err := cmd.Collect() - if err != nil { - log.Printf("Failed to list mailboxes with pattern '%s': %v", pattern, err) - continue - } - - for _, info := range infos { - if !seen[info.Mailbox] { - allMailboxes = append(allMailboxes, info.Mailbox) - seen[info.Mailbox] = true - } - } - } - - // Apply exclude filters client-side (IMAP LIST doesn't support exclusion) - if len(filter.Exclude) == 0 { - return allMailboxes, nil - } - - var filteredMailboxes []string - for _, mailbox := range allMailboxes { - excluded := false - for _, excludePattern := range filter.Exclude { - if matched := c.matchesImapPattern(excludePattern, mailbox); matched { - excluded = true - break - } - } - if !excluded { - filteredMailboxes = append(filteredMailboxes, mailbox) - } - } - - return filteredMailboxes, nil -} - -// matchesImapPattern matches IMAP-style patterns (simple * wildcard matching) -func (c *ImapClient) matchesImapPattern(pattern, name string) bool { - // Handle exact match - if pattern == name { - return true - } - - // Handle simple prefix wildcard: "Work*" should match "Work/Projects" - if strings.HasSuffix(pattern, "*") && !strings.Contains(pattern[:len(pattern)-1], "*") { - prefix := strings.TrimSuffix(pattern, "*") - return strings.HasPrefix(name, prefix) - } - - // Handle simple suffix wildcard: "*Temp" should match "Work/Temp" - if strings.HasPrefix(pattern, "*") && !strings.Contains(pattern[1:], "*") { - suffix := strings.TrimPrefix(pattern, "*") - return strings.HasSuffix(name, suffix) - } - - // Handle contains wildcard: "*Temp*" should match "Work/Temp/Archive" - if strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*") { - middle := strings.Trim(pattern, "*") - return strings.Contains(name, middle) - } - - // For other patterns, fall back to basic string comparison - return false -} - // GetMessages retrieves messages from a specific mailbox with filtering support // Returns messages and a map of all current UIDs in the mailbox // maxMessages: 0 means no limit, > 0 limits the number of messages to fetch @@ -458,6 +381,47 @@ func (c *ImapClient) parseMessagePart(entity *message.Entity, msg *Message) erro return nil } +// ShouldProcessMailbox checks if a mailbox should be processed based on filters with wildcard support +func (c *ImapClient) ShouldProcessMailbox(mailbox string, filter *config.FolderFilter) bool { + // If include list is specified, mailbox must match at least one pattern + if len(filter.Include) > 0 { + found := false + for _, pattern := range filter.Include { + // Handle special case: "*" means include all folders + if pattern == "*" { + found = true + break + } + // Use filepath.Match for wildcard pattern matching + if matched, err := filepath.Match(pattern, mailbox); err == nil && matched { + found = true + break + } + // Also support exact string matching for backwards compatibility + if mailbox == pattern { + found = true + break + } + } + if !found { + return false + } + } + + // If exclude list is specified, mailbox must not match any exclude pattern + for _, pattern := range filter.Exclude { + // Use filepath.Match for wildcard pattern matching + if matched, err := filepath.Match(pattern, mailbox); err == nil && matched { + return false + } + // Also support exact string matching for backwards compatibility + if mailbox == pattern { + return false + } + } + + return true +} // ShouldProcessMessage checks if a message should be processed based on keyword filters func (c *ImapClient) ShouldProcessMessage(msg *Message, filter *config.MessageFilter) bool { diff --git a/go/mail2couch b/go/mail2couch deleted file mode 100755 index cd300c5..0000000 Binary files a/go/mail2couch and /dev/null differ diff --git a/go/main.go b/go/main.go index 399d67c..155b195 100644 --- a/go/main.go +++ b/go/main.go @@ -66,13 +66,12 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN fmt.Println(" IMAP connection successful.") - // Use IMAP LIST with patterns for server-side filtering - mailboxes, err := imapClient.ListFilteredMailboxes(&source.FolderFilter) + mailboxes, err := imapClient.ListMailboxes() if err != nil { - return fmt.Errorf("failed to list filtered mailboxes: %w", err) + return fmt.Errorf("failed to list mailboxes: %w", err) } - fmt.Printf(" Found %d matching mailboxes after filtering.\n", len(mailboxes)) + fmt.Printf(" Found %d mailboxes.\n", len(mailboxes)) // Parse the since date from config if provided (fallback for first sync) var configSinceDate *time.Time @@ -88,8 +87,14 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN totalMessages := 0 totalStored := 0 - // Process each mailbox (already filtered by IMAP LIST) + // Process each mailbox for _, mailbox := range mailboxes { + // Check if this mailbox should be processed based on filters + if !imapClient.ShouldProcessMailbox(mailbox, &source.FolderFilter) { + fmt.Printf(" Skipping mailbox: %s (filtered)\n", mailbox) + continue + } + fmt.Printf(" Processing mailbox: %s (mode: %s)\n", mailbox, source.Mode) // Get sync metadata to determine incremental sync date diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b10a27e..fbdd2a4 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -36,9 +36,6 @@ async-std = { version = "1.12", features = ["attributes"] } # TLS support for secure IMAP connections async-native-tls = "0.5" -# Email parsing with MIME support -mail-parser = "0.6" - # Logging log = "0.4" env_logger = "0.10" @@ -52,9 +49,6 @@ dirs = "5.0" # Pattern matching for folder filters glob = "0.3" -# URL encoding for document IDs -urlencoding = "2.1" - [dev-dependencies] # Testing utilities tokio-test = "0.4" diff --git a/rust/config-test-rust-no-filter.json b/rust/config-test-rust-no-filter.json deleted file mode 100644 index 518c5cd..0000000 --- a/rust/config-test-rust-no-filter.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "couchDb": { - "url": "http://localhost:5984", - "user": "admin", - "password": "password" - }, - "mailSources": [ - { - "name": "Rust Wildcard All Folders Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser1", - "password": "password123", - "mode": "archive", - "folderFilter": { - "include": ["*"], - "exclude": ["Drafts", "Trash"] - }, - "messageFilter": {} - }, - { - "name": "Rust Work Pattern Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "syncuser", - "password": "syncpass", - "mode": "sync", - "folderFilter": { - "include": ["Work*", "Important*", "INBOX"], - "exclude": ["*Temp*"] - }, - "messageFilter": {} - }, - { - "name": "Rust Specific Folders Only", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "archiveuser", - "password": "archivepass", - "mode": "archive", - "folderFilter": { - "include": ["INBOX", "Sent", "Personal"], - "exclude": [] - }, - "messageFilter": {} - } - ] -} \ No newline at end of file diff --git a/rust/config-test-rust-single.json b/rust/config-test-rust-single.json deleted file mode 100644 index 292bfc9..0000000 --- a/rust/config-test-rust-single.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "couchDb": { - "url": "http://localhost:5984", - "user": "admin", - "password": "password" - }, - "mailSources": [ - { - "name": "Rust Wildcard All Folders Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser1", - "password": "password123", - "mode": "archive", - "folderFilter": { - "include": ["*"], - "exclude": ["Drafts", "Trash"] - }, - "messageFilter": {} - } - ] -} \ No newline at end of file diff --git a/rust/config-test-rust.json b/rust/config-test-rust.json deleted file mode 100644 index f323acc..0000000 --- a/rust/config-test-rust.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "couchDb": { - "url": "http://localhost:5984", - "user": "admin", - "password": "password" - }, - "mailSources": [ - { - "name": "Rust Wildcard All Folders Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser1", - "password": "password123", - "mode": "archive", - "folderFilter": { - "include": ["*"], - "exclude": ["Drafts", "Trash"] - }, - "messageFilter": { - "subjectKeywords": ["meeting", "important"], - "senderKeywords": ["@company.com"] - } - }, - { - "name": "Rust Work Pattern Test", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "syncuser", - "password": "syncpass", - "mode": "sync", - "folderFilter": { - "include": ["Work*", "Important*", "INBOX"], - "exclude": ["*Temp*"] - }, - "messageFilter": { - "recipientKeywords": ["support@", "team@"] - } - }, - { - "name": "Rust Specific Folders Only", - "enabled": true, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "archiveuser", - "password": "archivepass", - "mode": "archive", - "folderFilter": { - "include": ["INBOX", "Sent", "Personal"], - "exclude": [] - }, - "messageFilter": {} - }, - { - "name": "Rust Subfolder Pattern Test", - "enabled": false, - "protocol": "imap", - "host": "localhost", - "port": 3143, - "user": "testuser2", - "password": "password456", - "mode": "archive", - "folderFilter": { - "include": ["Work/*", "Archive/*"], - "exclude": ["*/Drafts"] - }, - "messageFilter": {} - } - ] -} \ No newline at end of file diff --git a/rust/src/config.rs b/rust/src/config.rs index 209a120..7744849 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -77,15 +77,6 @@ pub struct MessageFilter { pub recipient_keywords: Vec, } -impl MessageFilter { - /// Check if this filter has any keyword-based filters that can use IMAP SEARCH - pub fn has_keyword_filters(&self) -> bool { - !self.subject_keywords.is_empty() || !self.sender_keywords.is_empty() - // Note: recipient_keywords not included as IMAP SEARCH doesn't have a TO field search - // that works reliably across all IMAP servers - } -} - fn default_mode() -> String { "archive".to_string() } diff --git a/rust/src/couch.rs b/rust/src/couch.rs index 881dc99..e7dbd5d 100644 --- a/rust/src/couch.rs +++ b/rust/src/couch.rs @@ -8,7 +8,6 @@ use crate::schemas::{MailDocument, SyncMetadata}; use anyhow::{anyhow, Result}; use reqwest::{Client, StatusCode}; use serde_json::Value; -use std::time::Duration; use thiserror::Error; #[derive(Error, Debug)] @@ -43,62 +42,6 @@ pub struct CouchResponse { } impl CouchClient { - /// Generic retry helper for CouchDB operations - async fn retry_operation(&self, operation_name: &str, operation: F) -> Result - where - F: Fn() -> Fut, - Fut: std::future::Future>, - { - const MAX_RETRIES: u32 = 3; - const RETRY_DELAY_MS: u64 = 1000; - - let mut last_error = None; - - for attempt in 1..=MAX_RETRIES { - match operation().await { - Ok(result) => { - if attempt > 1 { - log::debug!("โœ… CouchDB {} successful on attempt {}", operation_name, attempt); - } - return Ok(result); - } - Err(e) => { - // Check if this is a retryable error - let is_retryable = match &e.downcast_ref::() { - Some(CouchError::Http(_)) => true, // Network errors are retryable - Some(CouchError::CouchDb { status, .. }) => { - // Retry on server errors (5xx) but not client errors (4xx) - *status >= 500 - } - _ => false, // Other errors are not retryable - }; - - last_error = Some(e); - - if is_retryable && attempt < MAX_RETRIES { - log::warn!( - "๐Ÿ”„ CouchDB {} attempt {} failed, retrying in {}ms: {}", - operation_name, - attempt, - RETRY_DELAY_MS, - last_error.as_ref().unwrap() - ); - tokio::time::sleep(Duration::from_millis(RETRY_DELAY_MS)).await; - } else { - break; - } - } - } - } - - Err(anyhow!( - "CouchDB {} failed after {} attempts. Last error: {}", - operation_name, - MAX_RETRIES, - last_error.unwrap() - )) - } - /// Create a new CouchDB client pub fn new(config: &CouchDbConfig) -> Result { let client = Client::new(); @@ -172,71 +115,22 @@ impl CouchClient { Ok(response.status().is_success()) } - /// Store a mail document in CouchDB with optional attachments and retry logic + /// Store a mail document in CouchDB pub async fn store_mail_document(&self, db_name: &str, mut document: MailDocument) -> Result { // Set the document ID if not already set if document.id.is_none() { document.set_id(); } - let doc_id = document.id.as_ref().unwrap().clone(); + let doc_id = document.id.as_ref().unwrap(); // Check if document already exists to avoid duplicates - if self.document_exists(db_name, &doc_id).await? { - return Ok(doc_id); + if self.document_exists(db_name, doc_id).await? { + return Ok(doc_id.clone()); } - self.retry_operation("store_mail_document", || async { - let encoded_doc_id = urlencoding::encode(&doc_id); - let url = format!("{}/{}/{}", self.base_url, db_name, encoded_doc_id); - let mut request = self.client.put(&url).json(&document); - - if let Some((username, password)) = &self.auth { - request = request.basic_auth(username, Some(password)); - } - - let response = request.send().await - .map_err(|e| CouchError::Http(e))?; - - match response.status() { - StatusCode::CREATED | StatusCode::ACCEPTED => { - let couch_response: CouchResponse = response.json().await - .map_err(|e| CouchError::Http(e))?; - Ok(couch_response.id.unwrap_or_else(|| doc_id.clone())) - } - status => { - let error_text = response.text().await - .unwrap_or_else(|_| "Failed to read error response".to_string()); - Err(CouchError::CouchDb { - status: status.as_u16(), - message: error_text, - }.into()) - } - } - }).await - } - - /// Store an attachment for a document in CouchDB - pub async fn store_attachment( - &self, - db_name: &str, - doc_id: &str, - attachment_name: &str, - content_type: &str, - data: &[u8], - ) -> Result { - // First get the current document revision - let doc_response = self.get_document_rev(db_name, doc_id).await?; - let rev = doc_response.ok_or_else(|| anyhow!("Document {} not found", doc_id))?; - - // Upload the attachment - let encoded_doc_id = urlencoding::encode(doc_id); - let encoded_attachment_name = urlencoding::encode(attachment_name); - let url = format!("{}/{}/{}/{}?rev={}", self.base_url, db_name, encoded_doc_id, encoded_attachment_name, rev); - let mut request = self.client - .put(&url) - .header("Content-Type", content_type) - .body(data.to_vec()); + let url = format!("{}/{}/{}", self.base_url, db_name, doc_id); + let mut request = self.client.put(&url).json(&document); if let Some((username, password)) = &self.auth { request = request.basic_auth(username, Some(password)); @@ -247,36 +141,11 @@ impl CouchClient { match response.status() { StatusCode::CREATED | StatusCode::ACCEPTED => { let couch_response: CouchResponse = response.json().await?; - Ok(couch_response.rev.unwrap_or_else(|| rev)) + Ok(couch_response.id.unwrap_or_else(|| doc_id.clone())) } status => { let error_text = response.text().await?; - Err(anyhow!("Failed to store attachment {}: {} - {}", attachment_name, status, error_text)) - } - } - } - - /// Get document revision - async fn get_document_rev(&self, db_name: &str, doc_id: &str) -> Result> { - let encoded_doc_id = urlencoding::encode(doc_id); - let url = format!("{}/{}/{}", self.base_url, db_name, encoded_doc_id); - let mut request = self.client.get(&url); - - if let Some((username, password)) = &self.auth { - request = request.basic_auth(username, Some(password)); - } - - let response = request.send().await?; - - match response.status() { - StatusCode::OK => { - let doc: Value = response.json().await?; - Ok(doc["_rev"].as_str().map(|s| s.to_string())) - } - StatusCode::NOT_FOUND => Ok(None), - status => { - let error_text = response.text().await?; - Err(anyhow!("Failed to get document {}: {} - {}", doc_id, status, error_text)) + Err(anyhow!("Failed to store document {}: {} - {}", doc_id, status, error_text)) } } } @@ -291,8 +160,7 @@ impl CouchClient { metadata_to_store.rev = existing.rev; } - let encoded_doc_id = urlencoding::encode(doc_id); - let url = format!("{}/{}/{}", self.base_url, db_name, encoded_doc_id); + let url = format!("{}/{}/{}", self.base_url, db_name, doc_id); let mut request = self.client.put(&url).json(&metadata_to_store); if let Some((username, password)) = &self.auth { @@ -316,8 +184,7 @@ impl CouchClient { /// Get sync metadata for a mailbox pub async fn get_sync_metadata(&self, db_name: &str, mailbox: &str) -> Result { let doc_id = format!("sync_metadata_{}", mailbox); - let encoded_doc_id = urlencoding::encode(&doc_id); - let url = format!("{}/{}/{}", self.base_url, db_name, encoded_doc_id); + let url = format!("{}/{}/{}", self.base_url, db_name, doc_id); let mut request = self.client.get(&url); if let Some((username, password)) = &self.auth { @@ -343,8 +210,7 @@ impl CouchClient { /// Check if a document exists pub async fn document_exists(&self, db_name: &str, doc_id: &str) -> Result { - let encoded_doc_id = urlencoding::encode(doc_id); - let url = format!("{}/{}/{}", self.base_url, db_name, encoded_doc_id); + let url = format!("{}/{}/{}", self.base_url, db_name, doc_id); let mut request = self.client.head(&url); if let Some((username, password)) = &self.auth { @@ -378,51 +244,10 @@ impl CouchClient { } } - /// Get all message UIDs for a specific mailbox from CouchDB - pub async fn get_mailbox_uids(&self, db_name: &str, mailbox: &str) -> Result> { - let url = format!("{}/{}/_all_docs", self.base_url, db_name); - let query_params = [ - ("startkey", format!("\"{}\"", mailbox)), - ("endkey", format!("\"{}\\ufff0\"", mailbox)), // High Unicode character for range end - ("include_docs", "false".to_string()), - ]; - - let mut request = self.client.get(&url).query(&query_params); - - if let Some((username, password)) = &self.auth { - request = request.basic_auth(username, Some(password)); - } - - let response = request.send().await?; - - if !response.status().is_success() { - return Err(anyhow!("Failed to query stored messages: {}", response.status())); - } - - let result: serde_json::Value = response.json().await?; - let mut uids = Vec::new(); - - if let Some(rows) = result["rows"].as_array() { - for row in rows { - if let Some(id) = row["id"].as_str() { - // Parse UID from document ID format: {mailbox}_{uid} - if let Some(uid_str) = id.strip_prefix(&format!("{}_", mailbox)) { - if let Ok(uid) = uid_str.parse::() { - uids.push(uid); - } - } - } - } - } - - Ok(uids) - } - /// Delete a document (used in sync mode for deleted messages) pub async fn delete_document(&self, db_name: &str, doc_id: &str) -> Result<()> { // First get the document to get its revision - let encoded_doc_id = urlencoding::encode(doc_id); - let url = format!("{}/{}/{}", self.base_url, db_name, encoded_doc_id); + let url = format!("{}/{}/{}", self.base_url, db_name, doc_id); let mut request = self.client.get(&url); if let Some((username, password)) = &self.auth { @@ -439,8 +264,8 @@ impl CouchClient { let rev = doc["_rev"].as_str() .ok_or_else(|| anyhow!("Document {} has no _rev field", doc_id))?; - // Now delete the document - let delete_url = format!("{}/{}/{}?rev={}", self.base_url, db_name, encoded_doc_id, rev); + // Now delete the document + let delete_url = format!("{}/{}/{}?rev={}", self.base_url, db_name, doc_id, rev); let mut delete_request = self.client.delete(&delete_url); if let Some((username, password)) = &self.auth { diff --git a/rust/src/imap.rs b/rust/src/imap.rs index 6b3d553..f5680dd 100644 --- a/rust/src/imap.rs +++ b/rust/src/imap.rs @@ -4,7 +4,7 @@ //! listing mailboxes, and retrieving messages. use crate::config::{MailSource, MessageFilter}; -use crate::schemas::{MailDocument, AttachmentStub}; +use crate::schemas::MailDocument; use anyhow::{anyhow, Result}; use async_imap::types::Fetch; use async_imap::{Client, Session}; @@ -14,10 +14,8 @@ use async_std::net::TcpStream; use async_std::stream::StreamExt; use async_std::task::{Context, Poll}; use chrono::{DateTime, Utc}; -use mail_parser::{Message, MimeHeaders}; use std::collections::HashMap; use std::pin::Pin; -use std::time::Duration; use thiserror::Error; #[derive(Error, Debug)] @@ -106,55 +104,17 @@ pub struct MailboxInfo { } impl ImapClient { - /// Create a new IMAP client and connect to the server with retry logic + /// Create a new IMAP client and connect to the server pub async fn connect(source: MailSource) -> Result { let mut client = ImapClient { session: None, source, }; - client.establish_connection_with_retry().await?; + client.establish_connection().await?; Ok(client) } - /// Establish connection with automatic retry logic - async fn establish_connection_with_retry(&mut self) -> Result<()> { - const MAX_RETRIES: u32 = 3; - const RETRY_DELAY_MS: u64 = 1000; - - let mut last_error = None; - - for attempt in 1..=MAX_RETRIES { - match self.establish_connection().await { - Ok(()) => { - if attempt > 1 { - log::info!("โœ… IMAP connection successful on attempt {}", attempt); - } - return Ok(()); - } - Err(e) => { - last_error = Some(e); - if attempt < MAX_RETRIES { - log::warn!( - "๐Ÿ”„ IMAP connection attempt {} failed, retrying in {}ms: {}", - attempt, - RETRY_DELAY_MS, - last_error.as_ref().unwrap() - ); - async_std::task::sleep(Duration::from_millis(RETRY_DELAY_MS)).await; - } - } - } - } - - Err(anyhow!( - "Failed to establish IMAP connection after {} attempts. Last error: {}", - MAX_RETRIES, - last_error.unwrap() - )) - } - - /// Establish connection to IMAP server async fn establish_connection(&mut self) -> Result<()> { // Connect to the IMAP server @@ -236,91 +196,6 @@ impl ImapClient { Ok(mailbox_names) } - /// List mailboxes using IMAP LIST with server-side pattern filtering - pub async fn list_filtered_mailboxes(&mut self, filter: &crate::config::FolderFilter) -> Result> { - let session = self.session.as_mut() - .ok_or_else(|| anyhow!("Not connected to IMAP server"))?; - - let mut all_mailboxes = Vec::new(); - let mut seen = std::collections::HashSet::new(); - - // If no include patterns, get all mailboxes - if filter.include.is_empty() { - return self.list_mailboxes().await; - } - - // Use IMAP LIST with each include pattern for server-side filtering - for pattern in &filter.include { - log::debug!("Listing mailboxes with pattern: {}", pattern); - - let mut mailboxes = session.list(Some(""), Some(pattern)).await - .map_err(|e| { - log::warn!("Failed to list mailboxes with pattern '{}': {:?}", pattern, e); - ImapError::Operation(format!("Failed to list mailboxes with pattern '{}': {:?}", pattern, e)) - })?; - - while let Some(mailbox_result) = mailboxes.next().await { - match mailbox_result { - Ok(mailbox) => { - let name = mailbox.name().to_string(); - if seen.insert(name.clone()) { - all_mailboxes.push(name); - } - } - Err(e) => { - log::warn!("Error processing mailbox with pattern '{}': {:?}", pattern, e); - continue; - } - } - } - } - - // Apply exclude filters client-side (IMAP LIST doesn't support exclusion) - if filter.exclude.is_empty() { - return Ok(all_mailboxes); - } - - let filtered_mailboxes: Vec = all_mailboxes - .into_iter() - .filter(|mailbox| { - !filter.exclude.iter().any(|exclude_pattern| { - self.matches_imap_pattern(exclude_pattern, mailbox) - }) - }) - .collect(); - - Ok(filtered_mailboxes) - } - - /// Match IMAP-style patterns (simple * wildcard matching for exclude filters) - fn matches_imap_pattern(&self, pattern: &str, name: &str) -> bool { - // Handle exact match - if pattern == name { - return true; - } - - // Handle simple prefix wildcard: "Work*" should match "Work/Projects" - if pattern.ends_with('*') && !pattern[..pattern.len()-1].contains('*') { - let prefix = &pattern[..pattern.len()-1]; - return name.starts_with(prefix); - } - - // Handle simple suffix wildcard: "*Temp" should match "Work/Temp" - if pattern.starts_with('*') && !pattern[1..].contains('*') { - let suffix = &pattern[1..]; - return name.ends_with(suffix); - } - - // Handle contains wildcard: "*Temp*" should match "Work/Temp/Archive" - if pattern.starts_with('*') && pattern.ends_with('*') { - let middle = &pattern[1..pattern.len()-1]; - return name.contains(middle); - } - - // For other patterns, fall back to basic string comparison - false - } - /// Select a mailbox pub async fn select_mailbox(&mut self, mailbox: &str) -> Result { let session = self.session.as_mut() @@ -338,132 +213,28 @@ impl ImapClient { }) } - /// Search for messages using IMAP SEARCH command with retry logic + /// Search for messages using IMAP SEARCH command /// Returns UIDs of matching messages pub async fn search_messages(&mut self, since_date: Option<&DateTime>) -> Result> { - const MAX_RETRIES: u32 = 3; - const RETRY_DELAY_MS: u64 = 500; - - let mut last_error = None; - - for attempt in 1..=MAX_RETRIES { - let result = self.search_messages_internal(since_date).await; - - match result { - Ok(uids) => { - if attempt > 1 { - log::debug!("โœ… IMAP search successful on attempt {}", attempt); - } - return Ok(uids); - } - Err(e) => { - last_error = Some(e); - if attempt < MAX_RETRIES { - log::warn!( - "๐Ÿ”„ IMAP search attempt {} failed, retrying in {}ms: {}", - attempt, - RETRY_DELAY_MS, - last_error.as_ref().unwrap() - ); - async_std::task::sleep(Duration::from_millis(RETRY_DELAY_MS)).await; - } - } - } - } - - Err(anyhow!( - "IMAP search failed after {} attempts. Last error: {}", - MAX_RETRIES, - last_error.unwrap() - )) - } - - /// Internal search implementation without retry logic - async fn search_messages_internal(&mut self, since_date: Option<&DateTime>) -> Result> { let session = self.session.as_mut() .ok_or_else(|| anyhow!("Not connected to IMAP server"))?; let search_query = if let Some(since) = since_date { // Format date as required by IMAP (DD-MMM-YYYY) - // IMAP months are 3-letter abbreviations in English let formatted_date = since.format("%d-%b-%Y").to_string(); - log::debug!("Searching for messages since: {}", formatted_date); format!("SINCE {}", formatted_date) } else { - log::debug!("Searching for all messages"); "ALL".to_string() }; - log::debug!("IMAP search query: {}", search_query); - let uids = session.uid_search(&search_query).await - .map_err(|e| ImapError::Operation(format!("Search failed with query '{}': {:?}", search_query, e)))?; + .map_err(|e| ImapError::Operation(format!("Search failed: {:?}", e)))?; - let uid_vec: Vec = uids.into_iter().collect(); - log::debug!("Found {} messages matching search criteria", uid_vec.len()); - - Ok(uid_vec) + Ok(uids.into_iter().collect()) } - /// Search for messages with advanced criteria - /// Supports multiple search parameters for more complex queries - pub async fn search_messages_advanced( - &mut self, - since_date: Option<&DateTime>, - before_date: Option<&DateTime>, - subject_keywords: Option<&[String]>, - from_keywords: Option<&[String]>, - ) -> Result> { - let session = self.session.as_mut() - .ok_or_else(|| anyhow!("Not connected to IMAP server"))?; - - let mut search_parts = Vec::new(); - - // Add date filters - if let Some(since) = since_date { - let formatted_date = since.format("%d-%b-%Y").to_string(); - search_parts.push(format!("SINCE {}", formatted_date)); - } - - if let Some(before) = before_date { - let formatted_date = before.format("%d-%b-%Y").to_string(); - search_parts.push(format!("BEFORE {}", formatted_date)); - } - - // Add subject keyword filters - if let Some(keywords) = subject_keywords { - for keyword in keywords { - search_parts.push(format!("SUBJECT \"{}\"", keyword.replace("\"", "\\\""))); - } - } - - // Add from keyword filters - if let Some(keywords) = from_keywords { - for keyword in keywords { - search_parts.push(format!("FROM \"{}\"", keyword.replace("\"", "\\\""))); - } - } - - // Build the final search query - let search_query = if search_parts.is_empty() { - "ALL".to_string() - } else { - search_parts.join(" ") - }; - - log::debug!("Advanced IMAP search query: {}", search_query); - - let uids = session.uid_search(&search_query).await - .map_err(|e| ImapError::Operation(format!("Advanced search failed with query '{}': {:?}", search_query, e)))?; - - let uid_vec: Vec = uids.into_iter().collect(); - log::debug!("Found {} messages matching advanced search criteria", uid_vec.len()); - - Ok(uid_vec) - } - - /// Fetch message by UID with attachment data - pub async fn fetch_message(&mut self, uid: u32, mailbox: &str) -> Result<(MailDocument, Vec<(String, String, Vec)>)> { + /// Fetch message by UID + pub async fn fetch_message(&mut self, uid: u32) -> Result { let session = self.session.as_mut() .ok_or_else(|| anyhow!("Not connected to IMAP server"))?; @@ -477,7 +248,7 @@ impl ImapClient { Ok(message) => { // Drop the messages stream to release the session borrow drop(messages); - self.parse_message(&message, uid, mailbox).await + self.parse_message(&message, uid).await } Err(e) => Err(ImapError::Operation(format!("Failed to process message {}: {:?}", uid, e)).into()), } @@ -486,8 +257,8 @@ impl ImapClient { } } - /// Fetch multiple messages by UIDs with attachment data - pub async fn fetch_messages(&mut self, uids: &[u32], max_count: Option, mailbox: &str) -> Result)>)>> { + /// Fetch multiple messages by UIDs + pub async fn fetch_messages(&mut self, uids: &[u32], max_count: Option) -> Result> { if uids.is_empty() { return Ok(Vec::new()); } @@ -531,8 +302,8 @@ impl ImapClient { let mut mail_documents = Vec::new(); for (i, message) in fetched_messages.iter().enumerate() { if let Some(&uid) = uids_to_fetch.get(i) { - match self.parse_message(message, uid, mailbox).await { - Ok((doc, attachments)) => mail_documents.push((doc, attachments)), + match self.parse_message(message, uid).await { + Ok(doc) => mail_documents.push(doc), Err(e) => { log::warn!("Failed to parse message {}: {}", uid, e); } @@ -543,276 +314,101 @@ impl ImapClient { Ok(mail_documents) } - /// Parse a raw IMAP message into a MailDocument with attachment data - async fn parse_message(&self, message: &Fetch, uid: u32, mailbox: &str) -> Result<(MailDocument, Vec<(String, String, Vec)>)> { + /// Parse a raw IMAP message into a MailDocument + async fn parse_message(&self, message: &Fetch, uid: u32) -> Result { let body = message.body() .ok_or_else(|| ImapError::Parsing("No message body found".to_string()))?; - // Parse the email using mail-parser library - let parsed_message = Message::parse(body) - .ok_or_else(|| ImapError::Parsing("Failed to parse email message".to_string()))?; + // Parse the email using a simple RFC822 parser + // This is a basic implementation - a production version would use a proper email parser + let email_str = String::from_utf8_lossy(body); + let (headers, body_content) = self.parse_rfc822(&email_str)?; - // Extract sender addresses - let from = self.extract_addresses(&parsed_message, "From"); - - // Extract recipient addresses - let to = self.extract_addresses(&parsed_message, "To"); + // Extract key fields + let from = self.parse_addresses(&headers, "from")?; + let to = self.parse_addresses(&headers, "to")?; + let subject = headers.get("subject") + .and_then(|v| v.first()) + .unwrap_or(&"No Subject".to_string()) + .clone(); - // Extract subject - let subject = parsed_message - .get_subject() - .unwrap_or("No Subject") - .to_string(); + // Parse date + let date = self.parse_date(&headers)?; - // Extract date - let date = if let Some(date_time) = parsed_message.get_date() { - DateTime::from_timestamp(date_time.to_timestamp(), 0).unwrap_or_else(|| Utc::now()) - } else { - Utc::now() - }; + // Get current mailbox name (this would need to be passed in properly) + let mailbox = "INBOX".to_string(); // Placeholder - should be passed from caller - // Extract body content (prefer text/plain, fallback to text/html) - let body_content = self.extract_body_content(&parsed_message); - - // Extract headers - let headers = self.extract_headers(&parsed_message); - - // Extract attachments and their data - let (has_attachments, attachment_stubs, attachment_data) = self.extract_attachments_with_data(&parsed_message); - - let mut mail_doc = MailDocument::new( + let mail_doc = MailDocument::new( uid.to_string(), - mailbox.to_string(), + mailbox, from, to, subject, date, body_content, headers, - has_attachments, + false, // TODO: Check for attachments ); - // Don't store attachment metadata in the document - // CouchDB will handle this when we store attachments separately - // This matches the Go implementation approach - - // Log attachment information - if !attachment_data.is_empty() { - log::info!("Found {} attachments for message {}", attachment_data.len(), uid); - } - - Ok((mail_doc, attachment_data)) + Ok(mail_doc) } - /// Extract email addresses from a parsed message - fn extract_addresses(&self, message: &Message, header_name: &str) -> Vec { - if let Some(header) = message.get_header(header_name) { - // For address headers, use as_text() and parse manually - // mail-parser doesn't provide a direct address parsing method - let header_text = header.as_text_ref().unwrap_or(""); - - // Simple address extraction - split by comma and clean up - header_text - .split(',') - .map(|addr| addr.trim().to_string()) - .filter(|addr| !addr.is_empty() && addr.contains('@')) - .collect() - } else { - Vec::new() - } - } - - /// Extract body content from a parsed message (prefer text/plain, fallback to text/html) - fn extract_body_content(&self, message: &Message) -> String { - // Try to get text/plain body first (index 0 = first text part) - if let Some(text_body) = message.get_text_body(0) { - return text_body.to_string(); - } - - // Fallback to HTML body if no plain text (index 0 = first HTML part) - if let Some(html_body) = message.get_html_body(0) { - return html_body.to_string(); - } - - // If neither standard method works, try to extract from parts manually - for part in &message.parts { - // Check content type for text parts - if let Some(content_type) = part.get_content_type() { - if content_type.c_type.starts_with("text/plain") { - if let Some(body) = part.get_text_contents() { - return body.to_string(); - } - } - } - } - - // Second pass for HTML parts if no plain text found - for part in &message.parts { - if let Some(content_type) = part.get_content_type() { - if content_type.c_type.starts_with("text/html") { - if let Some(body) = part.get_text_contents() { - return body.to_string(); - } - } - } - } - - // Last resort - try any text content - for part in &message.parts { - if let Some(body) = part.get_text_contents() { - if !body.trim().is_empty() { - return body.to_string(); - } - } - } - - // Absolutely last resort - empty body - "No body content found".to_string() - } - - /// Extract all headers from a parsed message - fn extract_headers(&self, message: &Message) -> HashMap> { + /// Basic RFC822 header and body parser + fn parse_rfc822(&self, email: &str) -> Result<(HashMap>, String)> { let mut headers = HashMap::new(); - - for header in message.get_headers() { - let name = header.name().to_lowercase(); - let value = match header.value().as_text_ref() { - Some(text) => text.to_string(), - None => format!("{:?}", header.value()), // Fallback for non-text values - }; - - headers.entry(name) - .or_insert_with(Vec::new) - .push(value); - } - - headers - } + let lines = email.lines(); + let mut body_lines = Vec::new(); + let mut in_body = false; - /// Extract attachments from a parsed message with binary data - /// Returns (has_attachments, attachment_stubs, attachment_data) - fn extract_attachments_with_data(&self, message: &Message) -> (bool, HashMap, Vec<(String, String, Vec)>) { - let mut attachment_stubs = HashMap::new(); - let mut attachment_data = Vec::new(); - - // Iterate through all message parts looking for attachments - for (index, part) in message.parts.iter().enumerate() { - // Check if this part is an attachment - if let Some(content_type) = part.get_content_type() { - let is_attachment = self.is_attachment_part(part, &content_type); + for line in lines { + if in_body { + body_lines.push(line); + } else if line.trim().is_empty() { + in_body = true; + } else if line.starts_with(' ') || line.starts_with('\t') { + // Continuation of previous header + // Skip for simplicity in this basic implementation + continue; + } else if let Some(colon_pos) = line.find(':') { + let header_name = line[..colon_pos].trim().to_lowercase(); + let header_value = line[colon_pos + 1..].trim().to_string(); - if is_attachment { - // Generate a filename for the attachment - let filename = self.get_attachment_filename(part, index); - - // Get the content data (try different methods based on content type) - let body_data = if let Some(text_content) = part.get_text_contents() { - // Text-based attachments - log::debug!("Found text attachment content: {} bytes", text_content.len()); - text_content.as_bytes().to_vec() - } else { - // For now, skip attachments without text content - // TODO: Implement binary attachment support with proper mail-parser API - log::debug!("Skipping non-text attachment for part {} (content-type: {})", index, content_type.c_type); - vec![] - }; - - let content_type_str = content_type.c_type.to_string(); - - // Only create attachment stub if we have actual data - if !body_data.is_empty() { - let attachment_stub = AttachmentStub { - content_type: content_type_str.clone(), - length: Some(body_data.len() as u64), - stub: None, // Will be stored as actual attachment data - }; - - attachment_stubs.insert(filename.clone(), attachment_stub); - attachment_data.push((filename, content_type_str, body_data)); - } - } + headers.entry(header_name) + .or_insert_with(Vec::new) + .push(header_value); } } - - let has_attachments = !attachment_stubs.is_empty(); - (has_attachments, attachment_stubs, attachment_data) + + let body = body_lines.join("\n"); + Ok((headers, body)) } - /// Extract attachments from a parsed message (deprecated - use extract_attachments_with_data) - /// Returns (has_attachments, attachment_stubs) - fn extract_attachments(&self, message: &Message) -> (bool, HashMap) { - let (has_attachments, attachment_stubs, _) = self.extract_attachments_with_data(message); - (has_attachments, attachment_stubs) - } - - /// Determine if a message part is an attachment - fn is_attachment_part(&self, part: &mail_parser::MessagePart, content_type: &mail_parser::ContentType) -> bool { - // Check Content-Disposition header first - if let Some(disposition) = part.get_content_disposition() { - return disposition.c_type.to_lowercase() == "attachment"; - } + /// Parse email addresses from headers + fn parse_addresses(&self, headers: &HashMap>, header_name: &str) -> Result> { + let addresses = headers.get(header_name) + .map(|values| values.clone()) + .unwrap_or_default(); - // If no explicit disposition, check content type - // Consider non-text types as potential attachments - let main_type = content_type.c_type.split('/').next().unwrap_or(""); - match main_type { - "text" => false, // Text parts are usually body content - "multipart" => false, // Multipart containers are not attachments - _ => true, // Images, applications, etc. are likely attachments - } + // Basic email extraction - just return the raw values for now + // A production implementation would properly parse RFC822 addresses + Ok(addresses) } - - /// Generate a filename for an attachment - fn get_attachment_filename(&self, part: &mail_parser::MessagePart, index: usize) -> String { - // Try to get filename from Content-Disposition - if let Some(disposition) = part.get_content_disposition() { - // Find filename in attributes vector - if let Some(attrs) = &disposition.attributes { - for (key, value) in attrs { - if key.to_lowercase() == "filename" { - return value.to_string(); - } - } - } - } - - // Try to get filename from Content-Type - if let Some(content_type) = part.get_content_type() { - // Find name in attributes vector - if let Some(attrs) = &content_type.attributes { - for (key, value) in attrs { - if key.to_lowercase() == "name" { - return value.to_string(); - } - } - } - } - - // Generate a default filename based on content type and index - if let Some(content_type) = part.get_content_type() { - let extension = self.get_extension_from_content_type(&content_type.c_type); - format!("attachment_{}{}", index, extension) - } else { - format!("attachment_{}.bin", index) - } - } - - /// Get file extension from MIME content type - fn get_extension_from_content_type(&self, content_type: &str) -> &'static str { - match content_type { - "image/jpeg" => ".jpg", - "image/png" => ".png", - "image/gif" => ".gif", - "application/pdf" => ".pdf", - "application/zip" => ".zip", - "application/msword" => ".doc", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => ".docx", - "application/vnd.ms-excel" => ".xls", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => ".xlsx", - "text/plain" => ".txt", - "text/html" => ".html", - _ => ".bin", // Default binary extension - } + + /// Parse date from headers + fn parse_date(&self, headers: &HashMap>) -> Result> { + let default_date = Utc::now().to_rfc2822(); + let date_str = headers.get("date") + .and_then(|v| v.first()) + .unwrap_or(&default_date); + + // Try to parse RFC2822 date format + // For simplicity, fall back to current time if parsing fails + DateTime::parse_from_rfc2822(date_str) + .map(|dt| dt.with_timezone(&Utc)) + .or_else(|_| { + log::warn!("Failed to parse date '{}', using current time", date_str); + Ok(Utc::now()) + }) } /// Close the IMAP connection @@ -927,8 +523,12 @@ mod tests { }, }; - // Test email parsing with the new mail-parser implementation - // This test needs to be updated to use actual message parsing - // For now, we'll skip the detailed test since it requires a full email message + let email = "From: sender@example.com\r\nTo: recipient@example.com\r\nSubject: Test\r\n\r\nTest body\r\n"; + let (headers, body) = client.parse_rfc822(email).unwrap(); + + assert_eq!(headers.get("from").unwrap()[0], "sender@example.com"); + assert_eq!(headers.get("to").unwrap()[0], "recipient@example.com"); + assert_eq!(headers.get("subject").unwrap()[0], "Test"); + assert_eq!(body.trim(), "Test body"); } } \ No newline at end of file diff --git a/rust/src/sync.rs b/rust/src/sync.rs index 8fdec27..9e261a6 100644 --- a/rust/src/sync.rs +++ b/rust/src/sync.rs @@ -5,7 +5,7 @@ use crate::config::{Config, MailSource, CommandLineArgs}; use crate::couch::CouchClient; -use crate::filters::{get_filter_summary, validate_folder_patterns}; +use crate::filters::{filter_folders, get_filter_summary, validate_folder_patterns}; use crate::imap::{ImapClient, should_process_message}; use crate::schemas::{SyncMetadata, generate_database_name}; use anyhow::{anyhow, Result}; @@ -26,7 +26,6 @@ pub struct MailboxSyncResult { pub messages_processed: u32, pub messages_stored: u32, pub messages_skipped: u32, - pub messages_deleted: u32, pub last_uid: Option, pub sync_time: DateTime, } @@ -125,27 +124,19 @@ impl SyncCoordinator { // Connect to IMAP server let mut imap_client = ImapClient::connect(source.clone()).await?; - // Use IMAP LIST with patterns for server-side filtering - let filtered_mailboxes = imap_client.list_filtered_mailboxes(&source.folder_filter).await?; - info!("Found {} matching mailboxes after server-side filtering", filtered_mailboxes.len()); + // Get list of available mailboxes + let all_mailboxes = imap_client.list_mailboxes().await?; + info!("Found {} total mailboxes", all_mailboxes.len()); - // For validation and summary, we still need the full list - let all_mailboxes = if !source.folder_filter.include.is_empty() || !source.folder_filter.exclude.is_empty() { - // Only fetch all mailboxes if we have filters (for logging/validation) - imap_client.list_mailboxes().await.unwrap_or_else(|_| Vec::new()) - } else { - filtered_mailboxes.clone() - }; + // Apply folder filtering + let filtered_mailboxes = filter_folders(&all_mailboxes, &source.folder_filter); + let filter_summary = get_filter_summary(&all_mailboxes, &filtered_mailboxes, &source.folder_filter); + info!("{}", filter_summary); - if !all_mailboxes.is_empty() { - let filter_summary = get_filter_summary(&all_mailboxes, &filtered_mailboxes, &source.folder_filter); - info!("{}", filter_summary); - - // Validate folder patterns and show warnings - let warnings = validate_folder_patterns(&source.folder_filter, &all_mailboxes); - for warning in warnings { - warn!("{}", warning); - } + // Validate folder patterns and show warnings + let warnings = validate_folder_patterns(&source.folder_filter, &all_mailboxes); + for warning in warnings { + warn!("{}", warning); } // Sync each filtered mailbox @@ -157,24 +148,13 @@ impl SyncCoordinator { match self.sync_mailbox(&mut imap_client, &db_name, mailbox, source).await { Ok(result) => { - if result.messages_deleted > 0 { - info!( - " โœ… {}: {} processed, {} stored, {} skipped, {} deleted", - result.mailbox, - result.messages_processed, - result.messages_stored, - result.messages_skipped, - result.messages_deleted - ); - } else { - info!( - " โœ… {}: {} processed, {} stored, {} skipped", - result.mailbox, - result.messages_processed, - result.messages_stored, - result.messages_skipped - ); - } + info!( + " โœ… {}: {} processed, {} stored, {} skipped", + result.mailbox, + result.messages_processed, + result.messages_stored, + result.messages_skipped + ); total_messages += result.messages_processed; mailbox_results.push(result); } @@ -230,46 +210,9 @@ impl SyncCoordinator { } }; - // Search for messages using server-side IMAP SEARCH with keyword filtering when possible - let message_uids = if source.message_filter.has_keyword_filters() { - // Use advanced IMAP SEARCH with keyword filtering - let subject_keywords = if source.message_filter.subject_keywords.is_empty() { - None - } else { - Some(source.message_filter.subject_keywords.as_slice()) - }; - let from_keywords = if source.message_filter.sender_keywords.is_empty() { - None - } else { - Some(source.message_filter.sender_keywords.as_slice()) - }; - - info!(" Using IMAP SEARCH with keyword filters"); - imap_client.search_messages_advanced( - since_date.as_ref(), - None, // before_date - subject_keywords, - from_keywords, - ).await? - } else { - // Use simple date-based search - imap_client.search_messages(since_date.as_ref()).await? - }; - info!(" Found {} messages matching search criteria", message_uids.len()); - - // Handle sync mode - check for deleted messages - let mut messages_deleted = 0; - if source.mode == "sync" { - messages_deleted = self.handle_deleted_messages(db_name, mailbox, &message_uids).await - .unwrap_or_else(|e| { - warn!(" Failed to handle deleted messages: {}", e); - 0 - }); - - if messages_deleted > 0 { - info!(" ๐Ÿ—‘๏ธ Deleted {} messages that no longer exist on server", messages_deleted); - } - } + // Search for messages + let message_uids = imap_client.search_messages(since_date.as_ref()).await?; + info!(" Found {} messages to process", message_uids.len()); if message_uids.is_empty() { return Ok(MailboxSyncResult { @@ -277,7 +220,6 @@ impl SyncCoordinator { messages_processed: 0, messages_stored: 0, messages_skipped: 0, - messages_deleted, last_uid: None, sync_time: start_time, }); @@ -296,13 +238,13 @@ impl SyncCoordinator { }; // Fetch and process messages - let messages = imap_client.fetch_messages(uids_to_process, self.args.max_messages, mailbox).await?; + let messages = imap_client.fetch_messages(uids_to_process, self.args.max_messages).await?; let mut messages_stored = 0; let mut messages_skipped = 0; let mut last_uid = None; - for (mail_doc, attachments) in messages { + for mail_doc in messages { // Apply message filters if !should_process_message(&mail_doc, &source.message_filter) { messages_skipped += 1; @@ -312,31 +254,10 @@ impl SyncCoordinator { // Extract UID before moving the document let uid_str = mail_doc.source_uid.clone(); - // Store the message document first + // Store the message match self.couch_client.store_mail_document(db_name, mail_doc).await { - Ok(doc_id) => { + Ok(_) => { messages_stored += 1; - - // Store attachments if any exist - if !attachments.is_empty() { - for (filename, content_type, data) in attachments { - match self.couch_client.store_attachment( - db_name, - &doc_id, - &filename, - &content_type, - &data, - ).await { - Ok(_) => { - debug!(" Stored attachment: {}", filename); - } - Err(e) => { - warn!(" Failed to store attachment {}: {}", filename, e); - } - } - } - } - // Parse UID from source_uid if let Ok(uid) = uid_str.parse::() { last_uid = Some(last_uid.map_or(uid, |prev: u32| prev.max(uid))); @@ -368,58 +289,11 @@ impl SyncCoordinator { messages_processed: uids_to_process.len() as u32, messages_stored, messages_skipped, - messages_deleted, last_uid, sync_time: start_time, }) } - /// Handle deleted messages in sync mode - /// Compares UIDs from IMAP server with stored messages in CouchDB - /// and deletes messages that no longer exist on the server - async fn handle_deleted_messages( - &mut self, - db_name: &str, - mailbox: &str, - current_server_uids: &[u32], - ) -> Result { - // Get all stored message UIDs for this mailbox from CouchDB - let stored_uids = self.get_stored_message_uids(db_name, mailbox).await?; - - if stored_uids.is_empty() { - return Ok(0); // No stored messages to delete - } - - // Find UIDs that exist in CouchDB but not on the server - let server_uid_set: std::collections::HashSet = current_server_uids.iter().cloned().collect(); - let mut deleted_count = 0; - - for stored_uid in stored_uids { - if !server_uid_set.contains(&stored_uid) { - // This message was deleted from the server, remove it from CouchDB - let doc_id = format!("{}_{}", mailbox, stored_uid); - - match self.couch_client.delete_document(db_name, &doc_id).await { - Ok(_) => { - debug!(" Deleted document: {}", doc_id); - deleted_count += 1; - } - Err(e) => { - warn!(" Failed to delete document {}: {}", doc_id, e); - } - } - } - } - - Ok(deleted_count) - } - - /// Get all stored message UIDs for a mailbox from CouchDB - async fn get_stored_message_uids(&self, db_name: &str, mailbox: &str) -> Result> { - // Use the CouchDB client method to get stored UIDs - self.couch_client.get_mailbox_uids(db_name, mailbox).await - } - /// Print summary of sync results pub fn print_sync_summary(&self, results: &[SourceSyncResult]) { info!("\n๐ŸŽ‰ Synchronization completed!"); diff --git a/test-both-implementations.sh b/test-both-implementations.sh deleted file mode 100755 index 4319a72..0000000 --- a/test-both-implementations.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Test script to run both Rust and Go implementations with their respective configs -# This demonstrates feature parity between the implementations - -set -e - -echo "๐Ÿงช Testing both Rust and Go implementations with identical configurations" -echo "==================================================" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Check if test environment is running -check_test_env() { - echo -e "${BLUE}๐Ÿ“ก Checking test environment...${NC}" - - if ! curl -s http://localhost:5984 >/dev/null; then - echo -e "${YELLOW}โš ๏ธ Test environment not running. Starting it...${NC}" - cd test - ./start-test-env.sh - cd .. - echo -e "${GREEN}โœ… Test environment started${NC}" - else - echo -e "${GREEN}โœ… Test environment is running${NC}" - fi -} - -# Build both implementations -build_implementations() { - echo -e "${BLUE}๐Ÿ”จ Building implementations...${NC}" - - # Build Go implementation - echo -e "${BLUE} Building Go implementation...${NC}" - cd go - go build -o mail2couch . - cd .. - echo -e "${GREEN} โœ… Go implementation built${NC}" - - # Build Rust implementation - echo -e "${BLUE} Building Rust implementation...${NC}" - cd rust - cargo build --release - cd .. - echo -e "${GREEN} โœ… Rust implementation built${NC}" -} - -# Run Go implementation -run_go() { - echo -e "${BLUE}๐Ÿฆฌ Running Go implementation...${NC}" - cd go - echo -e "${BLUE} Using config: config-test-go.json${NC}" - ./mail2couch -c config-test-go.json - cd .. - echo -e "${GREEN}โœ… Go implementation completed${NC}" -} - -# Run Rust implementation -run_rust() { - echo -e "${BLUE}๐Ÿฆ€ Running Rust implementation...${NC}" - cd rust - echo -e "${BLUE} Using config: config-test-rust.json${NC}" - ./target/release/mail2couch -c config-test-rust.json - cd .. - echo -e "${GREEN}โœ… Rust implementation completed${NC}" -} - -# Check results -check_results() { - echo -e "${BLUE}๐Ÿ” Checking results...${NC}" - - echo -e "${BLUE} Listing all databases:${NC}" - curl -s http://localhost:5984/_all_dbs | python3 -m json.tool - - echo -e "\n${BLUE} Go implementation databases:${NC}" - for db in go_wildcard_all_folders_test go_work_pattern_test go_specific_folders_only; do - db_name="m2c_${db}" - if curl -s "http://localhost:5984/${db_name}" >/dev/null 2>&1; then - doc_count=$(curl -s "http://localhost:5984/${db_name}" | python3 -c "import sys, json; print(json.load(sys.stdin).get('doc_count', 0))") - echo -e "${GREEN} โœ… ${db_name}: ${doc_count} documents${NC}" - else - echo -e "${RED} โŒ ${db_name}: not found${NC}" - fi - done - - echo -e "\n${BLUE} Rust implementation databases:${NC}" - for db in rust_wildcard_all_folders_test rust_work_pattern_test rust_specific_folders_only; do - db_name="m2c_${db}" - if curl -s "http://localhost:5984/${db_name}" >/dev/null 2>&1; then - doc_count=$(curl -s "http://localhost:5984/${db_name}" | python3 -c "import sys, json; print(json.load(sys.stdin).get('doc_count', 0))") - echo -e "${GREEN} โœ… ${db_name}: ${doc_count} documents${NC}" - else - echo -e "${RED} โŒ ${db_name}: not found${NC}" - fi - done -} - -# Main execution -main() { - echo -e "${YELLOW}๐Ÿš€ Starting feature parity test...${NC}" - - check_test_env - build_implementations - - echo -e "\n${YELLOW}๐Ÿ“Š Running implementations with identical configurations...${NC}" - run_go - echo "" - run_rust - - echo -e "\n${YELLOW}๐Ÿ“ˆ Checking results...${NC}" - check_results - - echo -e "\n${GREEN}๐ŸŽ‰ Feature parity test completed!${NC}" - echo -e "${BLUE}๐Ÿ’ก Both implementations should have created similar databases with identical message counts.${NC}" - echo -e "${BLUE}๐Ÿ”— View results at: http://localhost:5984/_utils${NC}" -} - -# Execute main function -main "$@" \ No newline at end of file diff --git a/test-config-comparison.md b/test-config-comparison.md deleted file mode 100644 index 90ae448..0000000 --- a/test-config-comparison.md +++ /dev/null @@ -1,154 +0,0 @@ -# Test Configuration Comparison: Rust vs Go - -## Overview - -Two identical test configurations have been created for testing both Rust and Go implementations with the test environment: - -- **Rust**: `/home/olemd/src/mail2couch/rust/config-test-rust.json` -- **Go**: `/home/olemd/src/mail2couch/go/config-test-go.json` - -## Configuration Details - -Both configurations use the **same test environment** from `/home/olemd/src/mail2couch/test/` with: - -### Database Connection -- **CouchDB URL**: `http://localhost:5984` -- **Admin Credentials**: `admin` / `password` - -### IMAP Test Server -- **Host**: `localhost` -- **Port**: `3143` (GreenMail test server) -- **Connection**: Plain (no TLS for testing) - -### Test Accounts - -Both configurations use the **same IMAP test accounts**: - -| Username | Password | Purpose | -|----------|----------|---------| -| `testuser1` | `password123` | Wildcard all folders test | -| `syncuser` | `syncpass` | Work pattern test (sync mode) | -| `archiveuser` | `archivepass` | Specific folders test | -| `testuser2` | `password456` | Subfolder pattern test (disabled) | - -### Mail Sources Configuration - -Both configurations define **identical mail sources** with only the account names differing: - -#### 1. Wildcard All Folders Test -- **Account Name**: "**Rust** Wildcard All Folders Test" vs "**Go** Wildcard All Folders Test" -- **Mode**: `archive` -- **Folders**: All folders (`*`) except `Drafts` and `Trash` -- **Filters**: Subject keywords: `["meeting", "important"]`, Sender keywords: `["@company.com"]` - -#### 2. Work Pattern Test -- **Account Name**: "**Rust** Work Pattern Test" vs "**Go** Work Pattern Test" -- **Mode**: `sync` (delete removed emails) -- **Folders**: `Work*`, `Important*`, `INBOX` (exclude `*Temp*`) -- **Filters**: Recipient keywords: `["support@", "team@"]` - -#### 3. Specific Folders Only -- **Account Name**: "**Rust** Specific Folders Only" vs "**Go** Specific Folders Only" -- **Mode**: `archive` -- **Folders**: Exactly `INBOX`, `Sent`, `Personal` -- **Filters**: None - -#### 4. Subfolder Pattern Test (Disabled) -- **Account Name**: "**Rust** Subfolder Pattern Test" vs "**Go** Subfolder Pattern Test" -- **Mode**: `archive` -- **Folders**: `Work/*`, `Archive/*` (exclude `*/Drafts`) -- **Status**: `enabled: false` - -## Expected Database Names - -When run, each implementation will create **different databases** due to the account name differences: - -### Rust Implementation Databases -- `m2c_rust_wildcard_all_folders_test` -- `m2c_rust_work_pattern_test` -- `m2c_rust_specific_folders_only` -- `m2c_rust_subfolder_pattern_test` (disabled) - -### Go Implementation Databases -- `m2c_go_wildcard_all_folders_test` -- `m2c_go_work_pattern_test` -- `m2c_go_specific_folders_only` -- `m2c_go_subfolder_pattern_test` (disabled) - -## Testing Commands - -### Start Test Environment -```bash -cd /home/olemd/src/mail2couch/test -./start-test-env.sh -``` - -### Run Rust Implementation -```bash -cd /home/olemd/src/mail2couch/rust -cargo build --release -./target/release/mail2couch -c config-test-rust.json -``` - -### Run Go Implementation -```bash -cd /home/olemd/src/mail2couch/go -go build -o mail2couch . -./mail2couch -c config-test-go.json -``` - -### Verify Results -```bash -# List all databases -curl http://localhost:5984/_all_dbs - -# Check Rust databases -curl http://localhost:5984/m2c_rust_wildcard_all_folders_test -curl http://localhost:5984/m2c_rust_work_pattern_test -curl http://localhost:5984/m2c_rust_specific_folders_only - -# Check Go databases -curl http://localhost:5984/m2c_go_wildcard_all_folders_test -curl http://localhost:5984/m2c_go_work_pattern_test -curl http://localhost:5984/m2c_go_specific_folders_only -``` - -### Stop Test Environment -```bash -cd /home/olemd/src/mail2couch/test -./stop-test-env.sh -``` - -## Validation Points - -Both implementations should produce **identical results** when processing the same IMAP accounts: - -1. **Database Structure**: Same document schemas and field names -2. **Message Processing**: Same email parsing and storage logic -3. **Folder Filtering**: Same wildcard pattern matching -4. **Message Filtering**: Same keyword filtering behavior -5. **Sync Behavior**: Same incremental sync and deletion handling -6. **Error Handling**: Same retry logic and error recovery - -The only differences should be: -- Database names (due to account name prefixes) -- Timestamp precision (implementation-specific) -- Internal document IDs format (if any) - -## Use Cases - -### Feature Parity Testing -Run both implementations with the same configuration to verify identical behavior: -```bash -# Run both implementations -./test-both-implementations.sh - -# Compare database contents -./compare-database-results.sh -``` - -### Performance Comparison -Use identical configurations to benchmark performance differences between Rust and Go implementations. - -### Development Testing -Use separate configurations during development to avoid database conflicts when testing both implementations simultaneously. \ No newline at end of file