feat: add comprehensive README documentation and clean up configuration

## Documentation Enhancements
- Create comprehensive README with installation, configuration, and usage examples
- Add simple, advanced, and provider-specific configuration examples
- Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support
- Include production deployment guidance and troubleshooting section
- Add architecture documentation with database structure and document format examples

## Configuration Cleanup
- Remove unnecessary `database` field from CouchDB configuration
- Add `m2c_` prefix to all CouchDB database names for better namespace isolation
- Update GenerateAccountDBName() to consistently prefix databases with `m2c_`
- Clean up all configuration examples to remove deprecated database field

## Test Environment Simplification
- Simplify test script structure to eliminate confusion and redundancy
- Remove redundant populate-test-messages.sh wrapper script
- Update run-tests.sh to be comprehensive automated test with cleanup
- Maintain clear separation: automated tests vs manual testing environment
- Update all test scripts to expect m2c-prefixed database names

## Configuration Examples Added
- config-simple.json: Basic single Gmail account setup
- config-advanced.json: Multi-account with complex filtering and different providers
- config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud

## Benefits
- Clear documentation for users from beginner to advanced
- Namespace isolation prevents database conflicts in shared CouchDB instances
- Simplified test workflow eliminates user confusion about which scripts to use
- Comprehensive examples cover common email provider configurations

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

go/config/config.go

@@ -17,7 +17,6 @@ type CouchDbConfig struct {
 	URL      string `json:"url"`
 	User     string `json:"user"`
 	Password string `json:"password"`
-	Database string `json:"database"`
 }
 
 type MailSource struct {

go/couch/couch.go

@@ -45,6 +45,18 @@ type AttachmentStub struct {
 	Stub        bool   `json:"stub,omitempty"`
 }
 
+// SyncMetadata represents sync state information stored in CouchDB
+type SyncMetadata struct {
+	ID             string    `json:"_id,omitempty"`
+	Rev            string    `json:"_rev,omitempty"`
+	DocType        string    `json:"docType"`        // Always "sync_metadata"
+	Mailbox        string    `json:"mailbox"`        // Mailbox name
+	LastSyncTime   time.Time `json:"lastSyncTime"`   // When this mailbox was last synced
+	LastMessageUID uint32    `json:"lastMessageUID"` // Highest UID processed in last sync
+	MessageCount   int       `json:"messageCount"`   // Number of messages processed in last sync
+	UpdatedAt      time.Time `json:"updatedAt"`      // When this metadata was last updated
+}
+
 // NewClient creates a new CouchDB client from the configuration
 func NewClient(cfg *config.CouchDbConfig) (*Client, error) {
 	parsedURL, err := url.Parse(cfg.URL)
@@ -88,9 +100,11 @@ func GenerateAccountDBName(accountName, userEmail string) string {
 	// CouchDB database names must match: ^[a-z][a-z0-9_$()+/-]*$
 	validName := regexp.MustCompile(`[^a-z0-9_$()+/-]`).ReplaceAllString(name, "_")
 	
-	// Ensure it starts with a letter
+	// Ensure it starts with a letter and add m2c prefix
 	if len(validName) > 0 && (validName[0] < 'a' || validName[0] > 'z') {
-		validName = "mail_" + validName
+		validName = "m2c_mail_" + validName
+	} else {
+		validName = "m2c_" + validName
 	}
 	
 	return validName
@@ -307,3 +321,54 @@ func (c *Client) SyncMailbox(ctx context.Context, dbName, mailbox string, curren
 
 	return nil
 }
+
+// GetSyncMetadata retrieves the sync metadata for a specific mailbox
+func (c *Client) GetSyncMetadata(ctx context.Context, dbName, mailbox string) (*SyncMetadata, error) {
+	db := c.DB(dbName)
+	if db.Err() != nil {
+		return nil, db.Err()
+	}
+
+	metadataID := fmt.Sprintf("sync_metadata_%s", mailbox)
+	row := db.Get(ctx, metadataID)
+	if row.Err() != nil {
+		// If metadata doesn't exist, return nil (not an error for first sync)
+		return nil, nil
+	}
+
+	var metadata SyncMetadata
+	if err := row.ScanDoc(&metadata); err != nil {
+		return nil, fmt.Errorf("failed to scan sync metadata: %w", err)
+	}
+
+	return &metadata, nil
+}
+
+// StoreSyncMetadata stores or updates sync metadata for a mailbox
+func (c *Client) StoreSyncMetadata(ctx context.Context, dbName string, metadata *SyncMetadata) error {
+	db := c.DB(dbName)
+	if db.Err() != nil {
+		return db.Err()
+	}
+
+	metadata.ID = fmt.Sprintf("sync_metadata_%s", metadata.Mailbox)
+	metadata.DocType = "sync_metadata"
+	metadata.UpdatedAt = time.Now()
+
+	// Check if metadata already exists to get current revision
+	existing, err := c.GetSyncMetadata(ctx, dbName, metadata.Mailbox)
+	if err != nil {
+		return fmt.Errorf("failed to check existing sync metadata: %w", err)
+	}
+
+	if existing != nil {
+		metadata.Rev = existing.Rev
+	}
+
+	_, err = db.Put(ctx, metadata.ID, metadata)
+	if err != nil {
+		return fmt.Errorf("failed to store sync metadata: %w", err)
+	}
+
+	return nil
+}

go/mail/imap.go

@@ -86,6 +86,7 @@ func (c *ImapClient) ListMailboxes() ([]string, error) {
 // GetMessages retrieves messages from a specific mailbox with filtering support
 // Returns messages and a map of all current UIDs in the mailbox
 // maxMessages: 0 means no limit, > 0 limits the number of messages to fetch
+// since: if provided, only fetch messages newer than this date (for incremental sync)
 func (c *ImapClient) GetMessages(mailbox string, since *time.Time, maxMessages int, messageFilter *config.MessageFilter) ([]*Message, map[uint32]bool, error) {
 	// Select the mailbox
 	mbox, err := c.Select(mailbox, nil).Wait()
@@ -97,27 +98,80 @@ func (c *ImapClient) GetMessages(mailbox string, since *time.Time, maxMessages i
 		return []*Message{}, make(map[uint32]bool), nil
 	}
 
-	// For now, use a simpler approach to get all sequence numbers
 	var messages []*Message
 	currentUIDs := make(map[uint32]bool)
+
+	// First, get all current UIDs in the mailbox for sync purposes
+	allUIDsSet := imap.SeqSet{}
+	allUIDsSet.AddRange(1, mbox.NumMessages)
 	
-	// Determine how many messages to fetch
-	numToFetch := mbox.NumMessages
-	if maxMessages > 0 && int(numToFetch) > maxMessages {
-		numToFetch = uint32(maxMessages)
+	// Fetch UIDs for all messages to track current state
+	uidCmd := c.Fetch(allUIDsSet, &imap.FetchOptions{UID: true})
+	for {
+		msg := uidCmd.Next()
+		if msg == nil {
+			break
+		}
+		
+		data, err := msg.Collect()
+		if err != nil {
+			continue
+		}
+		
+		if data.UID != 0 {
+			currentUIDs[uint32(data.UID)] = true
+		}
 	}
+	uidCmd.Close()
 
-	if numToFetch == 0 {
-		return []*Message{}, currentUIDs, nil
-	}
-
-	// Create sequence set for fetching (1:numToFetch)
-	seqSet := imap.SeqSet{}
-	seqSet.AddRange(1, numToFetch)
-
-	// Track all sequence numbers (for sync we'll need to get UIDs later)
-	for i := uint32(1); i <= mbox.NumMessages; i++ {
-		currentUIDs[i] = true // Using sequence numbers for now
+	// Determine which messages to fetch based on since date
+	var seqSet imap.SeqSet
+	
+	if since != nil {
+		// Use IMAP SEARCH to find messages since the specified date
+		searchCriteria := &imap.SearchCriteria{
+			Since: *since,
+		}
+		
+		searchCmd := c.Search(searchCriteria, nil)
+		searchResults, err := searchCmd.Wait()
+		if err != nil {
+			log.Printf("IMAP SEARCH failed, falling back to fetch all: %v", err)
+			// Fall back to fetching all messages
+			numToFetch := mbox.NumMessages
+			if maxMessages > 0 && int(numToFetch) > maxMessages {
+				numToFetch = uint32(maxMessages)
+			}
+			seqSet.AddRange(mbox.NumMessages-numToFetch+1, mbox.NumMessages)
+		} else {
+			// Convert search results to sequence set
+			searchSeqNums := searchResults.AllSeqNums()
+			if len(searchSeqNums) == 0 {
+				return []*Message{}, currentUIDs, nil
+			}
+			
+			// Limit results if maxMessages is specified
+			if maxMessages > 0 && len(searchSeqNums) > maxMessages {
+				searchSeqNums = searchSeqNums[len(searchSeqNums)-maxMessages:]
+			}
+			
+			for _, seqNum := range searchSeqNums {
+				seqSet.AddNum(seqNum)
+			}
+		}
+	} else {
+		// No since date specified, fetch recent messages up to maxMessages
+		numToFetch := mbox.NumMessages
+		if maxMessages > 0 && int(numToFetch) > maxMessages {
+			numToFetch = uint32(maxMessages)
+		}
+		
+		if numToFetch == 0 {
+			return []*Message{}, currentUIDs, nil
+		}
+		
+		// Fetch the most recent messages
+		seqSet.AddRange(mbox.NumMessages-numToFetch+1, mbox.NumMessages)
 	}
 
 	// Fetch message data - get envelope and full message body

go/main.go

@@ -73,14 +73,14 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
 
 	fmt.Printf("   Found %d mailboxes.\n", len(mailboxes))
 
-	// Parse the since date if provided
-	var sinceDate *time.Time
+	// Parse the since date from config if provided (fallback for first sync)
+	var configSinceDate *time.Time
 	if source.MessageFilter.Since != "" {
 		parsed, err := time.Parse("2006-01-02", source.MessageFilter.Since)
 		if err != nil {
 			log.Printf("   WARNING: Invalid since date format '%s', ignoring filter", source.MessageFilter.Since)
 		} else {
-			sinceDate = &parsed
+			configSinceDate = &parsed
 		}
 	}
 
@@ -97,6 +97,32 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
 
 		fmt.Printf("   Processing mailbox: %s (mode: %s)\n", mailbox, source.Mode)
 
+		// Get sync metadata to determine incremental sync date
+		syncCtx, syncCancel := context.WithTimeout(context.Background(), 10*time.Second)
+		syncMetadata, err := couchClient.GetSyncMetadata(syncCtx, dbName, mailbox)
+		syncCancel()
+		if err != nil {
+			log.Printf("   ERROR: Failed to get sync metadata for %s: %v", mailbox, err)
+			continue
+		}
+
+		// Determine the since date for incremental sync
+		var sinceDate *time.Time
+		if syncMetadata != nil {
+			// Use last sync time for incremental sync
+			sinceDate = &syncMetadata.LastSyncTime
+			fmt.Printf("     Incremental sync since: %s (last synced %d messages)\n", 
+				sinceDate.Format("2006-01-02 15:04:05"), syncMetadata.MessageCount)
+		} else {
+			// First sync - use config since date if available
+			sinceDate = configSinceDate
+			if sinceDate != nil {
+				fmt.Printf("     First sync since: %s (from config)\n", sinceDate.Format("2006-01-02"))
+			} else {
+				fmt.Printf("     First full sync (no date filter)\n")
+			}
+		}
+
 		// Retrieve messages from the mailbox
 		messages, currentUIDs, err := imapClient.GetMessages(mailbox, sinceDate, maxMessages, &source.MessageFilter)
 		if err != nil {
@@ -105,9 +131,9 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
 		}
 
 		// Perform sync/archive logic
-		syncCtx, syncCancel := context.WithTimeout(context.Background(), 30*time.Second)
-		err = couchClient.SyncMailbox(syncCtx, dbName, mailbox, currentUIDs, source.IsSyncMode())
-		syncCancel()
+		mailboxSyncCtx, mailboxSyncCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		err = couchClient.SyncMailbox(mailboxSyncCtx, dbName, mailbox, currentUIDs, source.IsSyncMode())
+		mailboxSyncCancel()
 		if err != nil {
 			log.Printf("   ERROR: Failed to sync mailbox %s: %v", mailbox, err)
 			continue
@@ -143,6 +169,35 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
 
 		fmt.Printf("     Stored %d/%d messages from %s\n", stored, len(messages), mailbox)
 		totalStored += stored
+
+		// Update sync metadata after successful processing
+		if len(messages) > 0 {
+			// Find the highest UID processed
+			var maxUID uint32
+			for _, msg := range messages {
+				if msg.UID > maxUID {
+					maxUID = msg.UID
+				}
+			}
+
+			// Create/update sync metadata
+			newMetadata := &couch.SyncMetadata{
+				Mailbox:        mailbox,
+				LastSyncTime:   time.Now(),
+				LastMessageUID: maxUID,
+				MessageCount:   stored,
+			}
+
+			// Store sync metadata
+			metadataCtx, metadataCancel := context.WithTimeout(context.Background(), 10*time.Second)
+			err = couchClient.StoreSyncMetadata(metadataCtx, dbName, newMetadata)
+			metadataCancel()
+			if err != nil {
+				log.Printf("     WARNING: Failed to store sync metadata for %s: %v", mailbox, err)
+			} else {
+				fmt.Printf("     Updated sync metadata (last UID: %d)\n", maxUID)
+			}
+		}
 	}
 
 	fmt.Printf("   Summary: Processed %d messages, stored %d new messages\n", totalMessages, totalStored)