feat: implement per-account databases and native CouchDB attachments

- Create separate CouchDB database for each mail source (account)
- Store email attachments as native CouchDB attachments
- Add GenerateAccountDBName() for CouchDB-compatible database naming
- Update MailDocument structure to support _attachments field
- Implement StoreAttachment() for CouchDB attachment API
- Add placeholder attachment testing for every 3rd message

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Ole-Morten Duesund 2025-08-01 16:12:17 +02:00
commit 79f19a8877
3 changed files with 161 additions and 53 deletions

View file

@ -3,7 +3,10 @@ package couch
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"net/url" "net/url"
"regexp"
"strings"
"time" "time"
"github.com/go-kivik/kivik/v4" "github.com/go-kivik/kivik/v4"
@ -19,18 +22,27 @@ type Client struct {
// MailDocument represents an email message stored in CouchDB // MailDocument represents an email message stored in CouchDB
type MailDocument struct { type MailDocument struct {
ID string `json:"_id,omitempty"` ID string `json:"_id,omitempty"`
Rev string `json:"_rev,omitempty"` Rev string `json:"_rev,omitempty"`
SourceUID string `json:"sourceUid"` // Unique ID from the mail source (e.g., IMAP UID) Attachments map[string]AttachmentStub `json:"_attachments,omitempty"` // CouchDB attachments
Mailbox string `json:"mailbox"` // Source mailbox name SourceUID string `json:"sourceUid"` // Unique ID from the mail source (e.g., IMAP UID)
From []string `json:"from"` Mailbox string `json:"mailbox"` // Source mailbox name
To []string `json:"to"` From []string `json:"from"`
Subject string `json:"subject"` To []string `json:"to"`
Date time.Time `json:"date"` Subject string `json:"subject"`
Body string `json:"body"` Date time.Time `json:"date"`
Headers map[string][]string `json:"headers"` Body string `json:"body"`
StoredAt time.Time `json:"storedAt"` // When the document was stored Headers map[string][]string `json:"headers"`
DocType string `json:"docType"` // Always "mail" StoredAt time.Time `json:"storedAt"` // When the document was stored
DocType string `json:"docType"` // Always "mail"
HasAttachments bool `json:"hasAttachments"` // Indicates if message has attachments
}
// AttachmentStub represents metadata for a CouchDB attachment
type AttachmentStub struct {
ContentType string `json:"content_type"`
Length int64 `json:"length,omitempty"`
Stub bool `json:"stub,omitempty"`
} }
// NewClient creates a new CouchDB client from the configuration // NewClient creates a new CouchDB client from the configuration
@ -63,27 +75,63 @@ func (c *Client) EnsureDB(ctx context.Context, dbName string) error {
return nil return nil
} }
// GenerateAccountDBName creates a CouchDB-compatible database name from account info
func GenerateAccountDBName(accountName, userEmail string) string {
// Use account name if available, otherwise fall back to email
name := accountName
if name == "" {
name = userEmail
}
// Convert to lowercase and replace invalid characters with underscores
name = strings.ToLower(name)
// CouchDB database names must match: ^[a-z][a-z0-9_$()+/-]*$
validName := regexp.MustCompile(`[^a-z0-9_$()+/-]`).ReplaceAllString(name, "_")
// Ensure it starts with a letter
if len(validName) > 0 && (validName[0] < 'a' || validName[0] > 'z') {
validName = "mail_" + validName
}
return validName
}
// ConvertMessage converts an IMAP message to a MailDocument // ConvertMessage converts an IMAP message to a MailDocument
func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument { func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument {
docID := fmt.Sprintf("%s_%d", mailbox, msg.UID) docID := fmt.Sprintf("%s_%d", mailbox, msg.UID)
return &MailDocument{ doc := &MailDocument{
ID: docID, ID: docID,
SourceUID: fmt.Sprintf("%d", msg.UID), SourceUID: fmt.Sprintf("%d", msg.UID),
Mailbox: mailbox, Mailbox: mailbox,
From: msg.From, From: msg.From,
To: msg.To, To: msg.To,
Subject: msg.Subject, Subject: msg.Subject,
Date: msg.Date, Date: msg.Date,
Body: msg.Body, Body: msg.Body,
Headers: msg.Headers, Headers: msg.Headers,
StoredAt: time.Now(), StoredAt: time.Now(),
DocType: "mail", DocType: "mail",
HasAttachments: len(msg.Attachments) > 0,
} }
// Prepare attachment metadata if attachments exist
if len(msg.Attachments) > 0 {
doc.Attachments = make(map[string]AttachmentStub)
for _, att := range msg.Attachments {
doc.Attachments[att.Filename] = AttachmentStub{
ContentType: att.ContentType,
Length: int64(len(att.Content)),
Stub: true,
}
}
}
return doc
} }
// StoreMessage stores a mail message in CouchDB // StoreMessage stores a mail message in CouchDB with attachments
func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocument) error { func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocument, msg *mail.Message) error {
db := c.DB(dbName) db := c.DB(dbName)
if db.Err() != nil { if db.Err() != nil {
return db.Err() return db.Err()
@ -99,19 +147,54 @@ func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocum
return nil // Document already exists, skip return nil // Document already exists, skip
} }
// Store the document // Store the document first (without attachments)
_, err = db.Put(ctx, doc.ID, doc) rev, err := db.Put(ctx, doc.ID, doc)
if err != nil { if err != nil {
return fmt.Errorf("failed to store document: %w", err) return fmt.Errorf("failed to store document: %w", err)
} }
// If there are attachments, store them as CouchDB attachments
if msg != nil && len(msg.Attachments) > 0 {
for _, att := range msg.Attachments {
err := c.StoreAttachment(ctx, dbName, doc.ID, rev, att.Filename, att.ContentType, att.Content)
if err != nil {
return fmt.Errorf("failed to store attachment %s: %w", att.Filename, err)
}
}
}
return nil return nil
} }
// StoreMessages stores multiple mail messages in CouchDB // StoreAttachment stores an attachment to an existing CouchDB document
func (c *Client) StoreMessages(ctx context.Context, dbName string, docs []*MailDocument) error { func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filename, contentType string, content []byte) error {
for _, doc := range docs { db := c.DB(dbName)
if err := c.StoreMessage(ctx, dbName, doc); err != nil { if db.Err() != nil {
return db.Err()
}
att := &kivik.Attachment{
Filename: filename,
ContentType: contentType,
Content: io.NopCloser(strings.NewReader(string(content))),
}
_, err := db.PutAttachment(ctx, docID, att, kivik.Rev(rev))
if err != nil {
return fmt.Errorf("failed to store attachment: %w", err)
}
return nil
}
// StoreMessages stores multiple mail messages in CouchDB with their corresponding attachments
func (c *Client) StoreMessages(ctx context.Context, dbName string, docs []*MailDocument, messages []*mail.Message) error {
for i, doc := range docs {
var msg *mail.Message
if i < len(messages) {
msg = messages[i]
}
if err := c.StoreMessage(ctx, dbName, doc, msg); err != nil {
return err return err
} }
} }

View file

@ -16,13 +16,21 @@ type ImapClient struct {
// Message represents an email message retrieved from IMAP // Message represents an email message retrieved from IMAP
type Message struct { type Message struct {
UID uint32 UID uint32
From []string From []string
To []string To []string
Subject string Subject string
Date time.Time Date time.Time
Body string Body string
Headers map[string][]string Headers map[string][]string
Attachments []Attachment
}
// Attachment represents an email attachment
type Attachment struct {
Filename string
ContentType string
Content []byte
} }
// NewImapClient creates a new IMAP client from the configuration // NewImapClient creates a new IMAP client from the configuration
@ -87,6 +95,18 @@ func (c *ImapClient) GetMessages(mailbox string, since *time.Time) ([]*Message,
Body: fmt.Sprintf("This is a placeholder message %d from mailbox %s", i, mailbox), Body: fmt.Sprintf("This is a placeholder message %d from mailbox %s", i, mailbox),
Headers: make(map[string][]string), Headers: make(map[string][]string),
} }
// Add a sample attachment for testing (every 3rd message)
if i%3 == 0 {
msg.Attachments = []Attachment{
{
Filename: fmt.Sprintf("sample_%d.txt", i),
ContentType: "text/plain",
Content: []byte(fmt.Sprintf("Sample attachment content for message %d", i)),
},
}
}
messages = append(messages, msg) messages = append(messages, msg)
} }

View file

@ -23,25 +23,30 @@ func main() {
log.Fatalf("Failed to create CouchDB client: %v", err) log.Fatalf("Failed to create CouchDB client: %v", err)
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err = couchClient.EnsureDB(ctx, cfg.CouchDb.Database)
if err != nil {
log.Printf("Could not ensure CouchDB database exists (is it running?): %v", err)
} else {
fmt.Printf("CouchDB database '%s' is ready.\n", cfg.CouchDb.Database)
}
fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources)) fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources))
for _, source := range cfg.MailSources { for _, source := range cfg.MailSources {
if !source.Enabled { if !source.Enabled {
continue continue
} }
// Generate per-account database name
dbName := couch.GenerateAccountDBName(source.Name, source.User)
// Ensure the account-specific database exists
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err = couchClient.EnsureDB(ctx, dbName)
cancel()
if err != nil {
log.Printf("Could not ensure CouchDB database '%s' exists (is it running?): %v", dbName, err)
continue
} else {
fmt.Printf("CouchDB database '%s' is ready for account: %s\n", dbName, source.Name)
}
fmt.Printf(" - Processing source: %s\n", source.Name) fmt.Printf(" - Processing source: %s\n", source.Name)
if source.Protocol == "imap" { if source.Protocol == "imap" {
err := processImapSource(&source, couchClient, cfg.CouchDb.Database) err := processImapSource(&source, couchClient, dbName)
if err != nil { if err != nil {
log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err) log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err)
} }
@ -112,11 +117,11 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
docs = append(docs, doc) docs = append(docs, doc)
} }
// Store messages in CouchDB // Store messages in CouchDB with attachments
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
stored := 0 stored := 0
for _, doc := range docs { for i, doc := range docs {
err := couchClient.StoreMessage(ctx, dbName, doc) err := couchClient.StoreMessage(ctx, dbName, doc, messages[i])
if err != nil { if err != nil {
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err) log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
} else { } else {