2025-07-29 17:18:20 +02:00
|
|
|
package main
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"fmt"
|
|
|
|
|
"log"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"mail2couch/config"
|
|
|
|
|
"mail2couch/couch"
|
|
|
|
|
"mail2couch/mail"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func main() {
|
2025-08-01 17:04:10 +02:00
|
|
|
args := config.ParseCommandLine()
|
2025-08-02 14:57:51 +02:00
|
|
|
|
2025-08-01 17:04:10 +02:00
|
|
|
cfg, err := config.LoadConfigWithDiscovery(args)
|
2025-07-29 17:18:20 +02:00
|
|
|
if err != nil {
|
|
|
|
|
log.Fatalf("Failed to load configuration: %v", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Initialize CouchDB client
|
|
|
|
|
couchClient, err := couch.NewClient(&cfg.CouchDb)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Fatalf("Failed to create CouchDB client: %v", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources))
|
|
|
|
|
for _, source := range cfg.MailSources {
|
|
|
|
|
if !source.Enabled {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-01 16:12:17 +02:00
|
|
|
// Generate per-account database name
|
|
|
|
|
dbName := couch.GenerateAccountDBName(source.Name, source.User)
|
2025-08-02 14:57:51 +02:00
|
|
|
|
2025-08-01 16:12:17 +02:00
|
|
|
// Ensure the account-specific database exists
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
|
err = couchClient.EnsureDB(ctx, dbName)
|
|
|
|
|
cancel()
|
2025-08-02 14:57:51 +02:00
|
|
|
|
2025-08-01 16:12:17 +02:00
|
|
|
if err != nil {
|
|
|
|
|
log.Printf("Could not ensure CouchDB database '%s' exists (is it running?): %v", dbName, err)
|
|
|
|
|
continue
|
|
|
|
|
} else {
|
|
|
|
|
fmt.Printf("CouchDB database '%s' is ready for account: %s\n", dbName, source.Name)
|
|
|
|
|
}
|
|
|
|
|
|
2025-07-29 17:18:20 +02:00
|
|
|
fmt.Printf(" - Processing source: %s\n", source.Name)
|
|
|
|
|
if source.Protocol == "imap" {
|
2025-08-01 17:04:10 +02:00
|
|
|
err := processImapSource(&source, couchClient, dbName, args.MaxMessages)
|
2025-07-29 17:18:20 +02:00
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-01 17:04:10 +02:00
|
|
|
func processImapSource(source *config.MailSource, couchClient *couch.Client, dbName string, maxMessages int) error {
|
2025-07-29 17:18:20 +02:00
|
|
|
fmt.Printf(" Connecting to IMAP server: %s:%d\n", source.Host, source.Port)
|
|
|
|
|
imapClient, err := mail.NewImapClient(source)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return fmt.Errorf("failed to connect to IMAP server: %w", err)
|
|
|
|
|
}
|
|
|
|
|
defer imapClient.Logout()
|
|
|
|
|
|
|
|
|
|
fmt.Println(" IMAP connection successful.")
|
|
|
|
|
|
|
|
|
|
mailboxes, err := imapClient.ListMailboxes()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return fmt.Errorf("failed to list mailboxes: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fmt.Printf(" Found %d mailboxes.\n", len(mailboxes))
|
|
|
|
|
|
feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples
## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field
## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names
## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud
## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
|
|
|
// Parse the since date from config if provided (fallback for first sync)
|
|
|
|
|
var configSinceDate *time.Time
|
2025-07-29 17:18:20 +02:00
|
|
|
if source.MessageFilter.Since != "" {
|
|
|
|
|
parsed, err := time.Parse("2006-01-02", source.MessageFilter.Since)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" WARNING: Invalid since date format '%s', ignoring filter", source.MessageFilter.Since)
|
|
|
|
|
} else {
|
feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples
## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field
## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names
## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud
## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
|
|
|
configSinceDate = &parsed
|
2025-07-29 17:18:20 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
totalMessages := 0
|
|
|
|
|
totalStored := 0
|
|
|
|
|
|
|
|
|
|
// Process each mailbox
|
|
|
|
|
for _, mailbox := range mailboxes {
|
|
|
|
|
// Check if this mailbox should be processed based on filters
|
|
|
|
|
if !imapClient.ShouldProcessMailbox(mailbox, &source.FolderFilter) {
|
|
|
|
|
fmt.Printf(" Skipping mailbox: %s (filtered)\n", mailbox)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-01 17:04:10 +02:00
|
|
|
fmt.Printf(" Processing mailbox: %s (mode: %s)\n", mailbox, source.Mode)
|
2025-07-29 17:18:20 +02:00
|
|
|
|
feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples
## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field
## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names
## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud
## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
|
|
|
// Get sync metadata to determine incremental sync date
|
|
|
|
|
syncCtx, syncCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
|
|
|
syncMetadata, err := couchClient.GetSyncMetadata(syncCtx, dbName, mailbox)
|
|
|
|
|
syncCancel()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" ERROR: Failed to get sync metadata for %s: %v", mailbox, err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Determine the since date for incremental sync
|
|
|
|
|
var sinceDate *time.Time
|
|
|
|
|
if syncMetadata != nil {
|
|
|
|
|
// Use last sync time for incremental sync
|
|
|
|
|
sinceDate = &syncMetadata.LastSyncTime
|
2025-08-02 14:57:51 +02:00
|
|
|
fmt.Printf(" Incremental sync since: %s (last synced %d messages)\n",
|
feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples
## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field
## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names
## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud
## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
|
|
|
sinceDate.Format("2006-01-02 15:04:05"), syncMetadata.MessageCount)
|
|
|
|
|
} else {
|
|
|
|
|
// First sync - use config since date if available
|
|
|
|
|
sinceDate = configSinceDate
|
|
|
|
|
if sinceDate != nil {
|
|
|
|
|
fmt.Printf(" First sync since: %s (from config)\n", sinceDate.Format("2006-01-02"))
|
|
|
|
|
} else {
|
|
|
|
|
fmt.Printf(" First full sync (no date filter)\n")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2025-07-29 17:18:20 +02:00
|
|
|
// Retrieve messages from the mailbox
|
feat: implement comprehensive wildcard folder selection and keyword filtering
## Wildcard Folder Selection
- Add support for wildcard patterns (`*`, `?`, `[abc]`) using filepath.Match
- Implement special case: `"*"` selects ALL available folders
- Support for complex include/exclude pattern combinations
- Maintain backwards compatibility with exact string matching
- Enable subfolder pattern matching (e.g., `Work/*`, `*/Drafts`)
## Keyword Filtering
- Add SubjectKeywords, SenderKeywords, RecipientKeywords to MessageFilter config
- Implement case-insensitive keyword matching across message fields
- Support multiple keywords per filter type with inclusive OR logic
- Add ShouldProcessMessage method for message-level filtering
## Enhanced Test Environment
- Create comprehensive wildcard pattern test scenarios
- Add 12 test folders covering various pattern types: Work/*, Important/*, Archive/*, exact matches
- Implement dedicated wildcard test script (test-wildcard-patterns.sh)
- Update test configurations to demonstrate real-world wildcard usage patterns
- Enhance test data generation with folder-specific messages for validation
## Documentation
- Create FOLDER_PATTERNS.md with comprehensive wildcard examples and use cases
- Update CLAUDE.md to reflect all implemented features and current status
- Enhance test README with detailed wildcard pattern explanations
- Provide configuration examples for common email organization scenarios
## Message Origin Tracking
- Verify all messages in CouchDB properly tagged with origin folder in `mailbox` field
- Maintain per-account database isolation for better organization
- Document ID format: `{folder}_{uid}` ensures uniqueness across folders
Key patterns supported:
- `["*"]` - All folders (with excludes)
- `["Work*", "Important*"]` - Prefix matching
- `["Work/*", "Archive/*"]` - Subfolder patterns
- `["INBOX", "Sent"]` - Exact matches
- Complex include/exclude combinations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 17:24:02 +02:00
|
|
|
messages, currentUIDs, err := imapClient.GetMessages(mailbox, sinceDate, maxMessages, &source.MessageFilter)
|
2025-07-29 17:18:20 +02:00
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" ERROR: Failed to get messages from %s: %v", mailbox, err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-01 17:04:10 +02:00
|
|
|
// Perform sync/archive logic
|
feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples
## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field
## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names
## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud
## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
|
|
|
mailboxSyncCtx, mailboxSyncCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
|
|
|
err = couchClient.SyncMailbox(mailboxSyncCtx, dbName, mailbox, currentUIDs, source.IsSyncMode())
|
|
|
|
|
mailboxSyncCancel()
|
2025-08-01 17:04:10 +02:00
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" ERROR: Failed to sync mailbox %s: %v", mailbox, err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
2025-07-29 17:18:20 +02:00
|
|
|
if len(messages) == 0 {
|
2025-08-01 17:04:10 +02:00
|
|
|
fmt.Printf(" No new messages found in %s\n", mailbox)
|
2025-07-29 17:18:20 +02:00
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fmt.Printf(" Found %d messages in %s\n", len(messages), mailbox)
|
|
|
|
|
totalMessages += len(messages)
|
|
|
|
|
|
|
|
|
|
// Convert messages to CouchDB documents
|
|
|
|
|
var docs []*couch.MailDocument
|
|
|
|
|
for _, msg := range messages {
|
|
|
|
|
doc := couch.ConvertMessage(msg, mailbox)
|
|
|
|
|
docs = append(docs, doc)
|
|
|
|
|
}
|
|
|
|
|
|
2025-08-01 16:12:17 +02:00
|
|
|
// Store messages in CouchDB with attachments
|
2025-07-29 17:18:20 +02:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
|
|
|
stored := 0
|
2025-08-01 16:12:17 +02:00
|
|
|
for i, doc := range docs {
|
|
|
|
|
err := couchClient.StoreMessage(ctx, dbName, doc, messages[i])
|
2025-07-29 17:18:20 +02:00
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
|
|
|
|
|
} else {
|
|
|
|
|
stored++
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
cancel()
|
|
|
|
|
|
|
|
|
|
fmt.Printf(" Stored %d/%d messages from %s\n", stored, len(messages), mailbox)
|
|
|
|
|
totalStored += stored
|
feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples
## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field
## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names
## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud
## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 21:26:53 +02:00
|
|
|
|
|
|
|
|
// Update sync metadata after successful processing
|
|
|
|
|
if len(messages) > 0 {
|
|
|
|
|
// Find the highest UID processed
|
|
|
|
|
var maxUID uint32
|
|
|
|
|
for _, msg := range messages {
|
|
|
|
|
if msg.UID > maxUID {
|
|
|
|
|
maxUID = msg.UID
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Create/update sync metadata
|
|
|
|
|
newMetadata := &couch.SyncMetadata{
|
|
|
|
|
Mailbox: mailbox,
|
|
|
|
|
LastSyncTime: time.Now(),
|
|
|
|
|
LastMessageUID: maxUID,
|
|
|
|
|
MessageCount: stored,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Store sync metadata
|
|
|
|
|
metadataCtx, metadataCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
|
|
|
err = couchClient.StoreSyncMetadata(metadataCtx, dbName, newMetadata)
|
|
|
|
|
metadataCancel()
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Printf(" WARNING: Failed to store sync metadata for %s: %v", mailbox, err)
|
|
|
|
|
} else {
|
|
|
|
|
fmt.Printf(" Updated sync metadata (last UID: %d)\n", maxUID)
|
|
|
|
|
}
|
|
|
|
|
}
|
2025-07-29 17:18:20 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fmt.Printf(" Summary: Processed %d messages, stored %d new messages\n", totalMessages, totalStored)
|
|
|
|
|
return nil
|
|
|
|
|
}
|