feat: implement per-account databases and native CouchDB attachments
- Create separate CouchDB database for each mail source (account) - Store email attachments as native CouchDB attachments - Add GenerateAccountDBName() for CouchDB-compatible database naming - Update MailDocument structure to support _attachments field - Implement StoreAttachment() for CouchDB attachment API - Add placeholder attachment testing for every 3rd message 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
44efed908d
commit
79f19a8877
3 changed files with 161 additions and 53 deletions
|
|
@ -3,7 +3,10 @@ package couch
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-kivik/kivik/v4"
|
"github.com/go-kivik/kivik/v4"
|
||||||
|
|
@ -21,6 +24,7 @@ type Client struct {
|
||||||
type MailDocument struct {
|
type MailDocument struct {
|
||||||
ID string `json:"_id,omitempty"`
|
ID string `json:"_id,omitempty"`
|
||||||
Rev string `json:"_rev,omitempty"`
|
Rev string `json:"_rev,omitempty"`
|
||||||
|
Attachments map[string]AttachmentStub `json:"_attachments,omitempty"` // CouchDB attachments
|
||||||
SourceUID string `json:"sourceUid"` // Unique ID from the mail source (e.g., IMAP UID)
|
SourceUID string `json:"sourceUid"` // Unique ID from the mail source (e.g., IMAP UID)
|
||||||
Mailbox string `json:"mailbox"` // Source mailbox name
|
Mailbox string `json:"mailbox"` // Source mailbox name
|
||||||
From []string `json:"from"`
|
From []string `json:"from"`
|
||||||
|
|
@ -31,6 +35,14 @@ type MailDocument struct {
|
||||||
Headers map[string][]string `json:"headers"`
|
Headers map[string][]string `json:"headers"`
|
||||||
StoredAt time.Time `json:"storedAt"` // When the document was stored
|
StoredAt time.Time `json:"storedAt"` // When the document was stored
|
||||||
DocType string `json:"docType"` // Always "mail"
|
DocType string `json:"docType"` // Always "mail"
|
||||||
|
HasAttachments bool `json:"hasAttachments"` // Indicates if message has attachments
|
||||||
|
}
|
||||||
|
|
||||||
|
// AttachmentStub represents metadata for a CouchDB attachment
|
||||||
|
type AttachmentStub struct {
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
Length int64 `json:"length,omitempty"`
|
||||||
|
Stub bool `json:"stub,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient creates a new CouchDB client from the configuration
|
// NewClient creates a new CouchDB client from the configuration
|
||||||
|
|
@ -63,11 +75,32 @@ func (c *Client) EnsureDB(ctx context.Context, dbName string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GenerateAccountDBName creates a CouchDB-compatible database name from account info
|
||||||
|
func GenerateAccountDBName(accountName, userEmail string) string {
|
||||||
|
// Use account name if available, otherwise fall back to email
|
||||||
|
name := accountName
|
||||||
|
if name == "" {
|
||||||
|
name = userEmail
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to lowercase and replace invalid characters with underscores
|
||||||
|
name = strings.ToLower(name)
|
||||||
|
// CouchDB database names must match: ^[a-z][a-z0-9_$()+/-]*$
|
||||||
|
validName := regexp.MustCompile(`[^a-z0-9_$()+/-]`).ReplaceAllString(name, "_")
|
||||||
|
|
||||||
|
// Ensure it starts with a letter
|
||||||
|
if len(validName) > 0 && (validName[0] < 'a' || validName[0] > 'z') {
|
||||||
|
validName = "mail_" + validName
|
||||||
|
}
|
||||||
|
|
||||||
|
return validName
|
||||||
|
}
|
||||||
|
|
||||||
// ConvertMessage converts an IMAP message to a MailDocument
|
// ConvertMessage converts an IMAP message to a MailDocument
|
||||||
func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument {
|
func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument {
|
||||||
docID := fmt.Sprintf("%s_%d", mailbox, msg.UID)
|
docID := fmt.Sprintf("%s_%d", mailbox, msg.UID)
|
||||||
|
|
||||||
return &MailDocument{
|
doc := &MailDocument{
|
||||||
ID: docID,
|
ID: docID,
|
||||||
SourceUID: fmt.Sprintf("%d", msg.UID),
|
SourceUID: fmt.Sprintf("%d", msg.UID),
|
||||||
Mailbox: mailbox,
|
Mailbox: mailbox,
|
||||||
|
|
@ -79,11 +112,26 @@ func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument {
|
||||||
Headers: msg.Headers,
|
Headers: msg.Headers,
|
||||||
StoredAt: time.Now(),
|
StoredAt: time.Now(),
|
||||||
DocType: "mail",
|
DocType: "mail",
|
||||||
|
HasAttachments: len(msg.Attachments) > 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare attachment metadata if attachments exist
|
||||||
|
if len(msg.Attachments) > 0 {
|
||||||
|
doc.Attachments = make(map[string]AttachmentStub)
|
||||||
|
for _, att := range msg.Attachments {
|
||||||
|
doc.Attachments[att.Filename] = AttachmentStub{
|
||||||
|
ContentType: att.ContentType,
|
||||||
|
Length: int64(len(att.Content)),
|
||||||
|
Stub: true,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// StoreMessage stores a mail message in CouchDB
|
return doc
|
||||||
func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocument) error {
|
}
|
||||||
|
|
||||||
|
// StoreMessage stores a mail message in CouchDB with attachments
|
||||||
|
func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocument, msg *mail.Message) error {
|
||||||
db := c.DB(dbName)
|
db := c.DB(dbName)
|
||||||
if db.Err() != nil {
|
if db.Err() != nil {
|
||||||
return db.Err()
|
return db.Err()
|
||||||
|
|
@ -99,19 +147,54 @@ func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocum
|
||||||
return nil // Document already exists, skip
|
return nil // Document already exists, skip
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store the document
|
// Store the document first (without attachments)
|
||||||
_, err = db.Put(ctx, doc.ID, doc)
|
rev, err := db.Put(ctx, doc.ID, doc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to store document: %w", err)
|
return fmt.Errorf("failed to store document: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If there are attachments, store them as CouchDB attachments
|
||||||
|
if msg != nil && len(msg.Attachments) > 0 {
|
||||||
|
for _, att := range msg.Attachments {
|
||||||
|
err := c.StoreAttachment(ctx, dbName, doc.ID, rev, att.Filename, att.ContentType, att.Content)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to store attachment %s: %w", att.Filename, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// StoreMessages stores multiple mail messages in CouchDB
|
// StoreAttachment stores an attachment to an existing CouchDB document
|
||||||
func (c *Client) StoreMessages(ctx context.Context, dbName string, docs []*MailDocument) error {
|
func (c *Client) StoreAttachment(ctx context.Context, dbName, docID, rev, filename, contentType string, content []byte) error {
|
||||||
for _, doc := range docs {
|
db := c.DB(dbName)
|
||||||
if err := c.StoreMessage(ctx, dbName, doc); err != nil {
|
if db.Err() != nil {
|
||||||
|
return db.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
att := &kivik.Attachment{
|
||||||
|
Filename: filename,
|
||||||
|
ContentType: contentType,
|
||||||
|
Content: io.NopCloser(strings.NewReader(string(content))),
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := db.PutAttachment(ctx, docID, att, kivik.Rev(rev))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to store attachment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StoreMessages stores multiple mail messages in CouchDB with their corresponding attachments
|
||||||
|
func (c *Client) StoreMessages(ctx context.Context, dbName string, docs []*MailDocument, messages []*mail.Message) error {
|
||||||
|
for i, doc := range docs {
|
||||||
|
var msg *mail.Message
|
||||||
|
if i < len(messages) {
|
||||||
|
msg = messages[i]
|
||||||
|
}
|
||||||
|
if err := c.StoreMessage(ctx, dbName, doc, msg); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,14 @@ type Message struct {
|
||||||
Date time.Time
|
Date time.Time
|
||||||
Body string
|
Body string
|
||||||
Headers map[string][]string
|
Headers map[string][]string
|
||||||
|
Attachments []Attachment
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attachment represents an email attachment
|
||||||
|
type Attachment struct {
|
||||||
|
Filename string
|
||||||
|
ContentType string
|
||||||
|
Content []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewImapClient creates a new IMAP client from the configuration
|
// NewImapClient creates a new IMAP client from the configuration
|
||||||
|
|
@ -87,6 +95,18 @@ func (c *ImapClient) GetMessages(mailbox string, since *time.Time) ([]*Message,
|
||||||
Body: fmt.Sprintf("This is a placeholder message %d from mailbox %s", i, mailbox),
|
Body: fmt.Sprintf("This is a placeholder message %d from mailbox %s", i, mailbox),
|
||||||
Headers: make(map[string][]string),
|
Headers: make(map[string][]string),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add a sample attachment for testing (every 3rd message)
|
||||||
|
if i%3 == 0 {
|
||||||
|
msg.Attachments = []Attachment{
|
||||||
|
{
|
||||||
|
Filename: fmt.Sprintf("sample_%d.txt", i),
|
||||||
|
ContentType: "text/plain",
|
||||||
|
Content: []byte(fmt.Sprintf("Sample attachment content for message %d", i)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
messages = append(messages, msg)
|
messages = append(messages, msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
33
go/main.go
33
go/main.go
|
|
@ -23,25 +23,30 @@ func main() {
|
||||||
log.Fatalf("Failed to create CouchDB client: %v", err)
|
log.Fatalf("Failed to create CouchDB client: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err = couchClient.EnsureDB(ctx, cfg.CouchDb.Database)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Could not ensure CouchDB database exists (is it running?): %v", err)
|
|
||||||
} else {
|
|
||||||
fmt.Printf("CouchDB database '%s' is ready.\n", cfg.CouchDb.Database)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources))
|
fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources))
|
||||||
for _, source := range cfg.MailSources {
|
for _, source := range cfg.MailSources {
|
||||||
if !source.Enabled {
|
if !source.Enabled {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Generate per-account database name
|
||||||
|
dbName := couch.GenerateAccountDBName(source.Name, source.User)
|
||||||
|
|
||||||
|
// Ensure the account-specific database exists
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
err = couchClient.EnsureDB(ctx, dbName)
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Could not ensure CouchDB database '%s' exists (is it running?): %v", dbName, err)
|
||||||
|
continue
|
||||||
|
} else {
|
||||||
|
fmt.Printf("CouchDB database '%s' is ready for account: %s\n", dbName, source.Name)
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Printf(" - Processing source: %s\n", source.Name)
|
fmt.Printf(" - Processing source: %s\n", source.Name)
|
||||||
if source.Protocol == "imap" {
|
if source.Protocol == "imap" {
|
||||||
err := processImapSource(&source, couchClient, cfg.CouchDb.Database)
|
err := processImapSource(&source, couchClient, dbName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err)
|
log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err)
|
||||||
}
|
}
|
||||||
|
|
@ -112,11 +117,11 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
|
||||||
docs = append(docs, doc)
|
docs = append(docs, doc)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store messages in CouchDB
|
// Store messages in CouchDB with attachments
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
stored := 0
|
stored := 0
|
||||||
for _, doc := range docs {
|
for i, doc := range docs {
|
||||||
err := couchClient.StoreMessage(ctx, dbName, doc)
|
err := couchClient.StoreMessage(ctx, dbName, doc, messages[i])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
|
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue