mail2couch/go/main.go

200 lines
6 KiB
Go
Raw Normal View History

package main
import (
"context"
"fmt"
"log"
"time"
"mail2couch/config"
"mail2couch/couch"
"mail2couch/mail"
)
func main() {
args := config.ParseCommandLine()
cfg, err := config.LoadConfigWithDiscovery(args)
if err != nil {
log.Fatalf("Failed to load configuration: %v", err)
}
// Initialize CouchDB client
couchClient, err := couch.NewClient(&cfg.CouchDb)
if err != nil {
log.Fatalf("Failed to create CouchDB client: %v", err)
}
fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources))
for _, source := range cfg.MailSources {
if !source.Enabled {
continue
}
// Generate per-account database name
dbName := couch.GenerateAccountDBName(source.Name, source.User)
// Ensure the account-specific database exists
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err = couchClient.EnsureDB(ctx, dbName)
cancel()
if err != nil {
log.Printf("Could not ensure CouchDB database '%s' exists (is it running?): %v", dbName, err)
continue
} else {
fmt.Printf("CouchDB database '%s' is ready for account: %s\n", dbName, source.Name)
}
fmt.Printf(" - Processing source: %s\n", source.Name)
if source.Protocol == "imap" {
err := processImapSource(&source, couchClient, dbName, args.MaxMessages)
if err != nil {
log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err)
}
}
}
}
func processImapSource(source *config.MailSource, couchClient *couch.Client, dbName string, maxMessages int) error {
fmt.Printf(" Connecting to IMAP server: %s:%d\n", source.Host, source.Port)
imapClient, err := mail.NewImapClient(source)
if err != nil {
return fmt.Errorf("failed to connect to IMAP server: %w", err)
}
defer imapClient.Logout()
fmt.Println(" IMAP connection successful.")
// Use IMAP LIST with patterns for server-side filtering
mailboxes, err := imapClient.ListFilteredMailboxes(&source.FolderFilter)
if err != nil {
return fmt.Errorf("failed to list filtered mailboxes: %w", err)
}
fmt.Printf(" Found %d matching mailboxes after filtering.\n", len(mailboxes))
feat: add comprehensive README documentation and clean up configuration ## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
// Parse the since date from config if provided (fallback for first sync)
var configSinceDate *time.Time
if source.MessageFilter.Since != "" {
parsed, err := time.Parse("2006-01-02", source.MessageFilter.Since)
if err != nil {
log.Printf(" WARNING: Invalid since date format '%s', ignoring filter", source.MessageFilter.Since)
} else {
feat: add comprehensive README documentation and clean up configuration ## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
configSinceDate = &parsed
}
}
totalMessages := 0
totalStored := 0
// Process each mailbox (already filtered by IMAP LIST)
for _, mailbox := range mailboxes {
fmt.Printf(" Processing mailbox: %s (mode: %s)\n", mailbox, source.Mode)
feat: add comprehensive README documentation and clean up configuration ## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
// Get sync metadata to determine incremental sync date
syncCtx, syncCancel := context.WithTimeout(context.Background(), 10*time.Second)
syncMetadata, err := couchClient.GetSyncMetadata(syncCtx, dbName, mailbox)
syncCancel()
if err != nil {
log.Printf(" ERROR: Failed to get sync metadata for %s: %v", mailbox, err)
continue
}
// Determine the since date for incremental sync
var sinceDate *time.Time
if syncMetadata != nil {
// Use last sync time for incremental sync
sinceDate = &syncMetadata.LastSyncTime
fmt.Printf(" Incremental sync since: %s (last synced %d messages)\n",
feat: add comprehensive README documentation and clean up configuration ## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
sinceDate.Format("2006-01-02 15:04:05"), syncMetadata.MessageCount)
} else {
// First sync - use config since date if available
sinceDate = configSinceDate
if sinceDate != nil {
fmt.Printf(" First sync since: %s (from config)\n", sinceDate.Format("2006-01-02"))
} else {
fmt.Printf(" First full sync (no date filter)\n")
}
}
// Retrieve messages from the mailbox
feat: implement comprehensive wildcard folder selection and keyword filtering ## Wildcard Folder Selection - Add support for wildcard patterns (`*`, `?`, `[abc]`) using filepath.Match - Implement special case: `"*"` selects ALL available folders - Support for complex include/exclude pattern combinations - Maintain backwards compatibility with exact string matching - Enable subfolder pattern matching (e.g., `Work/*`, `*/Drafts`) ## Keyword Filtering - Add SubjectKeywords, SenderKeywords, RecipientKeywords to MessageFilter config - Implement case-insensitive keyword matching across message fields - Support multiple keywords per filter type with inclusive OR logic - Add ShouldProcessMessage method for message-level filtering ## Enhanced Test Environment - Create comprehensive wildcard pattern test scenarios - Add 12 test folders covering various pattern types: Work/*, Important/*, Archive/*, exact matches - Implement dedicated wildcard test script (test-wildcard-patterns.sh) - Update test configurations to demonstrate real-world wildcard usage patterns - Enhance test data generation with folder-specific messages for validation ## Documentation - Create FOLDER_PATTERNS.md with comprehensive wildcard examples and use cases - Update CLAUDE.md to reflect all implemented features and current status - Enhance test README with detailed wildcard pattern explanations - Provide configuration examples for common email organization scenarios ## Message Origin Tracking - Verify all messages in CouchDB properly tagged with origin folder in `mailbox` field - Maintain per-account database isolation for better organization - Document ID format: `{folder}_{uid}` ensures uniqueness across folders Key patterns supported: - `["*"]` - All folders (with excludes) - `["Work*", "Important*"]` - Prefix matching - `["Work/*", "Archive/*"]` - Subfolder patterns - `["INBOX", "Sent"]` - Exact matches - Complex include/exclude combinations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 17:24:02 +02:00
messages, currentUIDs, err := imapClient.GetMessages(mailbox, sinceDate, maxMessages, &source.MessageFilter)
if err != nil {
log.Printf(" ERROR: Failed to get messages from %s: %v", mailbox, err)
continue
}
// Perform sync/archive logic
feat: add comprehensive README documentation and clean up configuration ## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
mailboxSyncCtx, mailboxSyncCancel := context.WithTimeout(context.Background(), 30*time.Second)
err = couchClient.SyncMailbox(mailboxSyncCtx, dbName, mailbox, currentUIDs, source.IsSyncMode())
mailboxSyncCancel()
if err != nil {
log.Printf(" ERROR: Failed to sync mailbox %s: %v", mailbox, err)
continue
}
if len(messages) == 0 {
fmt.Printf(" No new messages found in %s\n", mailbox)
continue
}
fmt.Printf(" Found %d messages in %s\n", len(messages), mailbox)
totalMessages += len(messages)
// Convert messages to CouchDB documents
var docs []*couch.MailDocument
for _, msg := range messages {
doc := couch.ConvertMessage(msg, mailbox)
docs = append(docs, doc)
}
// Store messages in CouchDB with attachments
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
stored := 0
for i, doc := range docs {
err := couchClient.StoreMessage(ctx, dbName, doc, messages[i])
if err != nil {
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
} else {
stored++
}
}
cancel()
fmt.Printf(" Stored %d/%d messages from %s\n", stored, len(messages), mailbox)
totalStored += stored
feat: add comprehensive README documentation and clean up configuration ## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
// Update sync metadata after successful processing
if len(messages) > 0 {
// Find the highest UID processed
var maxUID uint32
for _, msg := range messages {
if msg.UID > maxUID {
maxUID = msg.UID
}
}
// Create/update sync metadata
newMetadata := &couch.SyncMetadata{
Mailbox: mailbox,
LastSyncTime: time.Now(),
LastMessageUID: maxUID,
MessageCount: stored,
}
// Store sync metadata
metadataCtx, metadataCancel := context.WithTimeout(context.Background(), 10*time.Second)
err = couchClient.StoreSyncMetadata(metadataCtx, dbName, newMetadata)
metadataCancel()
if err != nil {
log.Printf(" WARNING: Failed to store sync metadata for %s: %v", mailbox, err)
} else {
fmt.Printf(" Updated sync metadata (last UID: %d)\n", maxUID)
}
}
}
fmt.Printf(" Summary: Processed %d messages, stored %d new messages\n", totalMessages, totalStored)
return nil
}