feat: implement real IMAP message parsing with native CouchDB attachments
- Replace placeholder message generation with actual IMAP message fetching using go-message library - Add per-account CouchDB databases for better organization and isolation - Implement native CouchDB attachment storage with proper revision management - Add command line argument parsing with --max-messages flag for controlling message processing limits - Support both sync and archive modes with proper document synchronization - Add comprehensive test environment with Podman containers (GreenMail IMAP server + CouchDB) - Implement full MIME multipart parsing for proper body and attachment extraction - Add TLS and plain IMAP connection support based on port configuration - Update configuration system to support sync vs archive modes - Create test scripts and sample data for development and testing Key technical improvements: - Real email envelope and header processing with go-imap v2 API - MIME Content-Type and Content-Disposition parsing for attachment detection - CouchDB document ID generation using mailbox_uid format for uniqueness - Duplicate detection and prevention to avoid re-storing existing messages - Proper error handling and connection management for IMAP operations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
79f19a8877
commit
ea6235b674
22 changed files with 1262 additions and 66 deletions
|
|
@ -115,17 +115,8 @@ func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument {
|
|||
HasAttachments: len(msg.Attachments) > 0,
|
||||
}
|
||||
|
||||
// Prepare attachment metadata if attachments exist
|
||||
if len(msg.Attachments) > 0 {
|
||||
doc.Attachments = make(map[string]AttachmentStub)
|
||||
for _, att := range msg.Attachments {
|
||||
doc.Attachments[att.Filename] = AttachmentStub{
|
||||
ContentType: att.ContentType,
|
||||
Length: int64(len(att.Content)),
|
||||
Stub: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
// Don't add attachment metadata here - CouchDB will handle this when we store attachments
|
||||
// We'll add the attachment metadata after successful document creation
|
||||
|
||||
return doc
|
||||
}
|
||||
|
|
@ -155,11 +146,13 @@ func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocum
|
|||
|
||||
// If there are attachments, store them as CouchDB attachments
|
||||
if msg != nil && len(msg.Attachments) > 0 {
|
||||
currentRev := rev
|
||||
for _, att := range msg.Attachments {
|
||||
err := c.StoreAttachment(ctx, dbName, doc.ID, rev, att.Filename, att.ContentType, att.Content)
|
||||
newRev, err := c.StoreAttachment(ctx, dbName, doc.ID, currentRev, att.Filename, att.ContentType, att.Content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to store attachment %s: %w", att.Filename, err)
|
||||
}
|
||||
currentRev = newRev // Update revision for next attachment
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -167,10 +160,10 @@ func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocum
|
|||
}
|
||||
|
||||
// StoreAttachment stores an attachment to an existing CouchDB document
|
||||
func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filename, contentType string, content []byte) error {
|
||||
func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filename, contentType string, content []byte) (string, error) {
|
||||
db := c.DB(dbName)
|
||||
if db.Err() != nil {
|
||||
return db.Err()
|
||||
return "", db.Err()
|
||||
}
|
||||
|
||||
att := &kivik.Attachment{
|
||||
|
|
@ -179,12 +172,12 @@ func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filena
|
|||
Content: io.NopCloser(strings.NewReader(string(content))),
|
||||
}
|
||||
|
||||
_, err := db.PutAttachment(ctx, docID, att, kivik.Rev(rev))
|
||||
newRev, err := db.PutAttachment(ctx, docID, att, kivik.Rev(rev))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to store attachment: %w", err)
|
||||
return "", fmt.Errorf("failed to store attachment: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return newRev, nil
|
||||
}
|
||||
|
||||
// StoreMessages stores multiple mail messages in CouchDB with their corresponding attachments
|
||||
|
|
@ -211,3 +204,106 @@ func (c *Client) DocumentExists(ctx context.Context, dbName, docID string) (bool
|
|||
row := db.Get(ctx, docID)
|
||||
return row.Err() == nil, nil
|
||||
}
|
||||
|
||||
// GetAllMailDocumentIDs returns all mail document IDs from a database for a specific mailbox
|
||||
func (c *Client) GetAllMailDocumentIDs(ctx context.Context, dbName, mailbox string) (map[string]bool, error) {
|
||||
db := c.DB(dbName)
|
||||
if db.Err() != nil {
|
||||
return nil, db.Err()
|
||||
}
|
||||
|
||||
// Create a view query to get all document IDs for the specified mailbox
|
||||
rows := db.AllDocs(ctx)
|
||||
|
||||
docIDs := make(map[string]bool)
|
||||
for rows.Next() {
|
||||
docID, err := rows.ID()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Filter by mailbox prefix (documents are named like "INBOX_123")
|
||||
if mailbox == "" || strings.HasPrefix(docID, mailbox+"_") {
|
||||
docIDs[docID] = true
|
||||
}
|
||||
}
|
||||
|
||||
if rows.Err() != nil {
|
||||
return nil, rows.Err()
|
||||
}
|
||||
|
||||
return docIDs, nil
|
||||
}
|
||||
|
||||
// DeleteDocument removes a document from CouchDB
|
||||
func (c *Client) DeleteDocument(ctx context.Context, dbName, docID string) error {
|
||||
db := c.DB(dbName)
|
||||
if db.Err() != nil {
|
||||
return db.Err()
|
||||
}
|
||||
|
||||
// Get the current revision
|
||||
row := db.Get(ctx, docID)
|
||||
if row.Err() != nil {
|
||||
return row.Err() // Document doesn't exist or other error
|
||||
}
|
||||
|
||||
var doc struct {
|
||||
Rev string `json:"_rev"`
|
||||
}
|
||||
if err := row.ScanDoc(&doc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete the document
|
||||
_, err := db.Delete(ctx, docID, doc.Rev)
|
||||
return err
|
||||
}
|
||||
|
||||
// SyncMailbox synchronizes a mailbox between mail server and CouchDB
|
||||
// In sync mode: removes documents from CouchDB that are no longer in the mail account
|
||||
// In archive mode: keeps all documents (no removal)
|
||||
func (c *Client) SyncMailbox(ctx context.Context, dbName, mailbox string, currentMessageUIDs map[uint32]bool, syncMode bool) error {
|
||||
if !syncMode {
|
||||
return nil // Archive mode - don't remove anything
|
||||
}
|
||||
|
||||
// Get all existing document IDs for this mailbox from CouchDB
|
||||
existingDocs, err := c.GetAllMailDocumentIDs(ctx, dbName, mailbox)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get existing documents: %w", err)
|
||||
}
|
||||
|
||||
// Find documents that should be removed (exist in CouchDB but not in mail account)
|
||||
var toDelete []string
|
||||
for docID := range existingDocs {
|
||||
// Extract UID from document ID (format: "mailbox_uid")
|
||||
parts := strings.Split(docID, "_")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
uidStr := parts[len(parts)-1]
|
||||
uid := uint32(0)
|
||||
if _, err := fmt.Sscanf(uidStr, "%d", &uid); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// If this UID is not in the current mail account, mark for deletion
|
||||
if !currentMessageUIDs[uid] {
|
||||
toDelete = append(toDelete, docID)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete documents that are no longer in the mail account
|
||||
for _, docID := range toDelete {
|
||||
if err := c.DeleteDocument(ctx, dbName, docID); err != nil {
|
||||
return fmt.Errorf("failed to delete document %s: %w", docID, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(toDelete) > 0 {
|
||||
fmt.Printf(" Sync mode: Removed %d documents no longer in mail account\n", len(toDelete))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue