feat: implement real IMAP message parsing with native CouchDB attachments

- Replace placeholder message generation with actual IMAP message fetching using go-message library
- Add per-account CouchDB databases for better organization and isolation
- Implement native CouchDB attachment storage with proper revision management
- Add command line argument parsing with --max-messages flag for controlling message processing limits
- Support both sync and archive modes with proper document synchronization
- Add comprehensive test environment with Podman containers (GreenMail IMAP server + CouchDB)
- Implement full MIME multipart parsing for proper body and attachment extraction
- Add TLS and plain IMAP connection support based on port configuration
- Update configuration system to support sync vs archive modes
- Create test scripts and sample data for development and testing

Key technical improvements:
- Real email envelope and header processing with go-imap v2 API
- MIME Content-Type and Content-Disposition parsing for attachment detection
- CouchDB document ID generation using mailbox_uid format for uniqueness
- Duplicate detection and prevention to avoid re-storing existing messages
- Proper error handling and connection management for IMAP operations

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ole-Morten Duesund 2025-08-01 17:04:10 +02:00
commit ea6235b674
22 changed files with 1262 additions and 66 deletions

View file

@ -115,17 +115,8 @@ func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument {
HasAttachments: len(msg.Attachments) > 0,
}
// Prepare attachment metadata if attachments exist
if len(msg.Attachments) > 0 {
doc.Attachments = make(map[string]AttachmentStub)
for _, att := range msg.Attachments {
doc.Attachments[att.Filename] = AttachmentStub{
ContentType: att.ContentType,
Length: int64(len(att.Content)),
Stub: true,
}
}
}
// Don't add attachment metadata here - CouchDB will handle this when we store attachments
// We'll add the attachment metadata after successful document creation
return doc
}
@ -155,11 +146,13 @@ func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocum
// If there are attachments, store them as CouchDB attachments
if msg != nil && len(msg.Attachments) > 0 {
currentRev := rev
for _, att := range msg.Attachments {
err := c.StoreAttachment(ctx, dbName, doc.ID, rev, att.Filename, att.ContentType, att.Content)
newRev, err := c.StoreAttachment(ctx, dbName, doc.ID, currentRev, att.Filename, att.ContentType, att.Content)
if err != nil {
return fmt.Errorf("failed to store attachment %s: %w", att.Filename, err)
}
currentRev = newRev // Update revision for next attachment
}
}
@ -167,10 +160,10 @@ func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocum
}
// StoreAttachment stores an attachment to an existing CouchDB document
func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filename, contentType string, content []byte) error {
func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filename, contentType string, content []byte) (string, error) {
db := c.DB(dbName)
if db.Err() != nil {
return db.Err()
return "", db.Err()
}
att := &kivik.Attachment{
@ -179,12 +172,12 @@ func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filena
Content: io.NopCloser(strings.NewReader(string(content))),
}
_, err := db.PutAttachment(ctx, docID, att, kivik.Rev(rev))
newRev, err := db.PutAttachment(ctx, docID, att, kivik.Rev(rev))
if err != nil {
return fmt.Errorf("failed to store attachment: %w", err)
return "", fmt.Errorf("failed to store attachment: %w", err)
}
return nil
return newRev, nil
}
// StoreMessages stores multiple mail messages in CouchDB with their corresponding attachments
@ -211,3 +204,106 @@ func (c *Client) DocumentExists(ctx context.Context, dbName, docID string) (bool
row := db.Get(ctx, docID)
return row.Err() == nil, nil
}
// GetAllMailDocumentIDs returns all mail document IDs from a database for a specific mailbox
func (c *Client) GetAllMailDocumentIDs(ctx context.Context, dbName, mailbox string) (map[string]bool, error) {
db := c.DB(dbName)
if db.Err() != nil {
return nil, db.Err()
}
// Create a view query to get all document IDs for the specified mailbox
rows := db.AllDocs(ctx)
docIDs := make(map[string]bool)
for rows.Next() {
docID, err := rows.ID()
if err != nil {
continue
}
// Filter by mailbox prefix (documents are named like "INBOX_123")
if mailbox == "" || strings.HasPrefix(docID, mailbox+"_") {
docIDs[docID] = true
}
}
if rows.Err() != nil {
return nil, rows.Err()
}
return docIDs, nil
}
// DeleteDocument removes a document from CouchDB
func (c *Client) DeleteDocument(ctx context.Context, dbName, docID string) error {
db := c.DB(dbName)
if db.Err() != nil {
return db.Err()
}
// Get the current revision
row := db.Get(ctx, docID)
if row.Err() != nil {
return row.Err() // Document doesn't exist or other error
}
var doc struct {
Rev string `json:"_rev"`
}
if err := row.ScanDoc(&doc); err != nil {
return err
}
// Delete the document
_, err := db.Delete(ctx, docID, doc.Rev)
return err
}
// SyncMailbox synchronizes a mailbox between mail server and CouchDB
// In sync mode: removes documents from CouchDB that are no longer in the mail account
// In archive mode: keeps all documents (no removal)
func (c *Client) SyncMailbox(ctx context.Context, dbName, mailbox string, currentMessageUIDs map[uint32]bool, syncMode bool) error {
if !syncMode {
return nil // Archive mode - don't remove anything
}
// Get all existing document IDs for this mailbox from CouchDB
existingDocs, err := c.GetAllMailDocumentIDs(ctx, dbName, mailbox)
if err != nil {
return fmt.Errorf("failed to get existing documents: %w", err)
}
// Find documents that should be removed (exist in CouchDB but not in mail account)
var toDelete []string
for docID := range existingDocs {
// Extract UID from document ID (format: "mailbox_uid")
parts := strings.Split(docID, "_")
if len(parts) < 2 {
continue
}
uidStr := parts[len(parts)-1]
uid := uint32(0)
if _, err := fmt.Sscanf(uidStr, "%d", &uid); err != nil {
continue
}
// If this UID is not in the current mail account, mark for deletion
if !currentMessageUIDs[uid] {
toDelete = append(toDelete, docID)
}
}
// Delete documents that are no longer in the mail account
for _, docID := range toDelete {
if err := c.DeleteDocument(ctx, dbName, docID); err != nil {
return fmt.Errorf("failed to delete document %s: %w", docID, err)
}
}
if len(toDelete) > 0 {
fmt.Printf(" Sync mode: Removed %d documents no longer in mail account\n", len(toDelete))
}
return nil
}