diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..1af233d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,121 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +mail2couch is a utility for backing up mail from various sources (primarily IMAP) to CouchDB. The project supports two implementations: +- **Go implementation**: Located in `/go/` directory (currently the active implementation) +- **Rust implementation**: Planned but not yet implemented + +## Development Commands + +### Go Implementation (Primary) + +```bash +# Build the application +cd go && go build -o mail2couch . + +# Run the application with automatic config discovery +cd go && ./mail2couch + +# Run with specific config file +cd go && ./mail2couch -config /path/to/config.json + +# Run linting/static analysis +cd go && go vet ./... + +# Run tests (currently no tests exist) +cd go && go test ./... + +# Check dependencies +cd go && go mod tidy +``` + +## Architecture + +### Core Components + +1. **Configuration (`config/`)**: JSON-based configuration system + - Supports multiple mail sources with filtering options + - CouchDB connection settings + - Each source can have folder and message filters + +2. **Mail Handling (`mail/`)**: IMAP client implementation + - Uses `github.com/emersion/go-imap/v2` for IMAP operations + - Supports TLS connections + - Currently only lists mailboxes (backup functionality not yet implemented) + +3. **CouchDB Integration (`couch/`)**: Database operations + - Uses `github.com/go-kivik/kivik/v4` as CouchDB driver + - Handles database creation and document management + - Defines `MailDocument` structure for email storage + +### Configuration Structure + +The application uses `config.json` for configuration with the following structure: +- `couchDb`: Database connection settings (URL, credentials, database name) +- `mailSources`: Array of mail sources with individual settings: + - Protocol support (currently only IMAP) + - Connection details (host, port, credentials) + - Filtering options for folders and messages + - Enable/disable per source + +### Configuration File Discovery + +The application automatically searches for configuration files in the following order: +1. Path specified by `-config` command line flag +2. `./config.json` (current working directory) +3. `./config/config.json` (config subdirectory) +4. `~/.config/mail2couch/config.json` (user XDG config directory) +5. `~/.mail2couch.json` (user home directory) + +This design ensures the same `config.json` format will work for both Go and Rust implementations. + +### Current Implementation Status + +- ✅ Configuration loading with automatic file discovery +- ✅ Command line flag support for config file path +- ✅ CouchDB client initialization and database creation +- ✅ IMAP connection and mailbox listing +- ✅ Build error fixes +- ✅ Email message retrieval framework (with placeholder data) +- ✅ Email storage to CouchDB framework +- ✅ Folder filtering logic +- ✅ Date filtering support +- ✅ Duplicate detection and prevention +- ❌ Real IMAP message parsing (currently uses placeholder data) +- ❌ Full message body and attachment handling +- ❌ Incremental sync functionality +- ❌ Rust implementation + +### Key Dependencies + +- `github.com/emersion/go-imap/v2`: IMAP client library +- `github.com/go-kivik/kivik/v4`: CouchDB client library + +### Development Notes + +- The main entry point is `main.go` which orchestrates the configuration loading, CouchDB setup, and mail source processing +- Each mail source is processed sequentially with proper error handling +- The application currently uses placeholder message data for testing the storage pipeline +- Message filtering by folder (include/exclude) and date (since) is implemented +- Duplicate detection prevents re-storing existing messages +- No tests are currently implemented +- The application uses automatic config file discovery as documented above + +### Next Steps + +To complete the implementation, the following items need to be addressed: + +1. **Real IMAP Message Parsing**: Replace placeholder message generation with actual IMAP message fetching and parsing using the correct go-imap/v2 API +2. **Message Body Extraction**: Implement proper text/plain and text/html body extraction from multipart messages +3. **Attachment Handling**: Add support for email attachments (optional) +4. **Error Recovery**: Add retry logic for network failures and partial sync recovery +5. **Performance**: Add batch operations for better CouchDB insertion performance +6. **Testing**: Add unit tests for all major components + +## Development Guidelines + +### Code Quality and Standards +- All code requires perfect linting and tool-formatting, exceptions are allowed only if documented properly \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..16a0c4f --- /dev/null +++ b/config.json @@ -0,0 +1,42 @@ +{ + "couchDb": { + "url": "http://localhost:5984", + "user": "admin", + "password": "password", + "database": "mail_backup" + }, + "mailSources": [ + { + "name": "Personal Gmail", + "enabled": true, + "protocol": "imap", + "host": "imap.gmail.com", + "port": 993, + "user": "your-email@gmail.com", + "password": "your-app-password", + "sync": true, + "folderFilter": { + "include": ["INBOX", "Sent"], + "exclude": ["Spam", "Trash"] + }, + "messageFilter": { + "since": "2024-01-01" + } + }, + { + "name": "Work Account", + "enabled": true, + "protocol": "imap", + "host": "imap.work.com", + "port": 993, + "user": "user@work.com", + "password": "password", + "sync": true, + "folderFilter": { + "include": [], + "exclude": [] + }, + "messageFilter": {} + } + ] +} diff --git a/go/config/config.go b/go/config/config.go new file mode 100644 index 0000000..6fc76f8 --- /dev/null +++ b/go/config/config.go @@ -0,0 +1,111 @@ +package config + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" +) + +type Config struct { + CouchDb CouchDbConfig `json:"couchDb"` + MailSources []MailSource `json:"mailSources"` +} + +type CouchDbConfig struct { + URL string `json:"url"` + User string `json:"user"` + Password string `json:"password"` + Database string `json:"database"` +} + +type MailSource struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Protocol string `json:"protocol"` + Host string `json:"host"` + Port int `json:"port"` + User string `json:"user"` + Password string `json:"password"` + Sync bool `json:"sync"` + FolderFilter FolderFilter `json:"folderFilter"` + MessageFilter MessageFilter `json:"messageFilter"` +} + +type FolderFilter struct { + Include []string `json:"include"` + Exclude []string `json:"exclude"` +} + +type MessageFilter struct { + Since string `json:"since,omitempty"` +} + +func LoadConfig(path string) (*Config, error) { + configFile, err := os.Open(path) + if err != nil { + return nil, err + } + defer configFile.Close() + + var config Config + jsonParser := json.NewDecoder(configFile) + if err = jsonParser.Decode(&config); err != nil { + return nil, err + } + + return &config, nil +} + +// FindConfigFile searches for config.json in the following order: +// 1. Path specified by -config flag +// 2. ./config.json (current directory) +// 3. ~/.config/mail2couch/config.json (user config directory) +// 4. ~/.mail2couch.json (user home directory) +func FindConfigFile() (string, error) { + // Check for command line flag + configFlag := flag.String("config", "", "Path to configuration file") + flag.Parse() + + if *configFlag != "" { + if _, err := os.Stat(*configFlag); err == nil { + return *configFlag, nil + } + return "", fmt.Errorf("specified config file not found: %s", *configFlag) + } + + // List of possible config file locations in order of preference + candidates := []string{ + "config.json", // Current directory + "config/config.json", // Config subdirectory + } + + // Add user directory paths + if homeDir, err := os.UserHomeDir(); err == nil { + candidates = append(candidates, + filepath.Join(homeDir, ".config", "mail2couch", "config.json"), + filepath.Join(homeDir, ".mail2couch.json"), + ) + } + + // Try each candidate location + for _, candidate := range candidates { + if _, err := os.Stat(candidate); err == nil { + return candidate, nil + } + } + + return "", fmt.Errorf("no configuration file found. Searched locations: %v", candidates) +} + +// LoadConfigWithDiscovery loads configuration using automatic file discovery +func LoadConfigWithDiscovery() (*Config, error) { + configPath, err := FindConfigFile() + if err != nil { + return nil, err + } + + fmt.Printf("Using configuration file: %s\n", configPath) + return LoadConfig(configPath) +} diff --git a/go/couch/couch.go b/go/couch/couch.go new file mode 100644 index 0000000..a8a9278 --- /dev/null +++ b/go/couch/couch.go @@ -0,0 +1,130 @@ +package couch + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/go-kivik/kivik/v4" + _ "github.com/go-kivik/kivik/v4/couchdb" // The CouchDB driver + "mail2couch/config" + "mail2couch/mail" +) + +// Client wraps the Kivik client +type Client struct { + *kivik.Client +} + +// MailDocument represents an email message stored in CouchDB +type MailDocument struct { + ID string `json:"_id,omitempty"` + Rev string `json:"_rev,omitempty"` + SourceUID string `json:"sourceUid"` // Unique ID from the mail source (e.g., IMAP UID) + Mailbox string `json:"mailbox"` // Source mailbox name + From []string `json:"from"` + To []string `json:"to"` + Subject string `json:"subject"` + Date time.Time `json:"date"` + Body string `json:"body"` + Headers map[string][]string `json:"headers"` + StoredAt time.Time `json:"storedAt"` // When the document was stored + DocType string `json:"docType"` // Always "mail" +} + +// NewClient creates a new CouchDB client from the configuration +func NewClient(cfg *config.CouchDbConfig) (*Client, error) { + parsedURL, err := url.Parse(cfg.URL) + if err != nil { + return nil, fmt.Errorf("invalid couchdb url: %w", err) + } + + parsedURL.User = url.UserPassword(cfg.User, cfg.Password) + dsn := parsedURL.String() + + client, err := kivik.New("couch", dsn) + if err != nil { + return nil, err + } + + return &Client{client}, nil +} + +// EnsureDB ensures that the configured database exists. +func (c *Client) EnsureDB(ctx context.Context, dbName string) error { + exists, err := c.DBExists(ctx, dbName) + if err != nil { + return err + } + if !exists { + return c.CreateDB(ctx, dbName) + } + return nil +} + +// ConvertMessage converts an IMAP message to a MailDocument +func ConvertMessage(msg *mail.Message, mailbox string) *MailDocument { + docID := fmt.Sprintf("%s_%d", mailbox, msg.UID) + + return &MailDocument{ + ID: docID, + SourceUID: fmt.Sprintf("%d", msg.UID), + Mailbox: mailbox, + From: msg.From, + To: msg.To, + Subject: msg.Subject, + Date: msg.Date, + Body: msg.Body, + Headers: msg.Headers, + StoredAt: time.Now(), + DocType: "mail", + } +} + +// StoreMessage stores a mail message in CouchDB +func (c *Client) StoreMessage(ctx context.Context, dbName string, doc *MailDocument) error { + db := c.DB(dbName) + if db.Err() != nil { + return db.Err() + } + + // Check if document already exists + exists, err := c.DocumentExists(ctx, dbName, doc.ID) + if err != nil { + return fmt.Errorf("failed to check if document exists: %w", err) + } + + if exists { + return nil // Document already exists, skip + } + + // Store the document + _, err = db.Put(ctx, doc.ID, doc) + if err != nil { + return fmt.Errorf("failed to store document: %w", err) + } + + return nil +} + +// StoreMessages stores multiple mail messages in CouchDB +func (c *Client) StoreMessages(ctx context.Context, dbName string, docs []*MailDocument) error { + for _, doc := range docs { + if err := c.StoreMessage(ctx, dbName, doc); err != nil { + return err + } + } + return nil +} + +// DocumentExists checks if a document with the given ID already exists. +func (c *Client) DocumentExists(ctx context.Context, dbName, docID string) (bool, error) { + db := c.DB(dbName) + if db.Err() != nil { + return false, db.Err() + } + + row := db.Get(ctx, docID) + return row.Err() == nil, nil +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..377160a --- /dev/null +++ b/go/go.mod @@ -0,0 +1,16 @@ +module mail2couch + +go 1.24.4 + +require ( + github.com/emersion/go-imap/v2 v2.0.0-beta.5 + github.com/go-kivik/kivik/v4 v4.4.0 +) + +require ( + github.com/emersion/go-message v0.18.1 // indirect + github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect + github.com/google/uuid v1.6.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.10.0 // indirect +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 0000000..7851761 --- /dev/null +++ b/go/go.sum @@ -0,0 +1,65 @@ +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emersion/go-imap/v2 v2.0.0-beta.5 h1:H3858DNmBuXyMK1++YrQIRdpKE1MwBc+ywBtg3n+0wA= +github.com/emersion/go-imap/v2 v2.0.0-beta.5/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk= +github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E= +github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 h1:hH4PQfOndHDlpzYfLAAfl63E8Le6F2+EL/cdhlkyRJY= +github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/go-kivik/kivik/v4 v4.4.0 h1:1YMqNvRMIIC+CJUtyldD7c4Czl6SqdUcnbusCoFOTfk= +github.com/go-kivik/kivik/v4 v4.4.0/go.mod h1:DnPzIEO7CcLOqJNuqxuo7EMZeK4bPsEbUSSmAfi+tL4= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 h1:nHoRIX8iXob3Y2kdt9KsjyIb7iApSvb3vgsd93xb5Ow= +github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0/go.mod h1:c1tRKs5Tx7E2+uHGSyyncziFjvGpgv4H2HrqXeUQ/Uk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +gitlab.com/flimzy/testy v0.14.0 h1:2nZV4Wa1OSJb3rOKHh0GJqvvhtE03zT+sKnPCI0owfQ= +gitlab.com/flimzy/testy v0.14.0/go.mod h1:m3aGuwdXc+N3QgnH+2Ar2zf1yg0UxNdIaXKvC5SlfMk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/mail/imap.go b/go/mail/imap.go new file mode 100644 index 0000000..7bad119 --- /dev/null +++ b/go/mail/imap.go @@ -0,0 +1,127 @@ +package mail + +import ( + "fmt" + "log" + "time" + + "github.com/emersion/go-imap/v2/imapclient" + "mail2couch/config" +) + +// ImapClient wraps the IMAP client +type ImapClient struct { + *imapclient.Client +} + +// Message represents an email message retrieved from IMAP +type Message struct { + UID uint32 + From []string + To []string + Subject string + Date time.Time + Body string + Headers map[string][]string +} + +// NewImapClient creates a new IMAP client from the configuration +func NewImapClient(source *config.MailSource) (*ImapClient, error) { + addr := fmt.Sprintf("%s:%d", source.Host, source.Port) + + client, err := imapclient.DialTLS(addr, nil) + if err != nil { + return nil, fmt.Errorf("failed to dial IMAP server: %w", err) + } + + if err := client.Login(source.User, source.Password).Wait(); err != nil { + return nil, fmt.Errorf("failed to login: %w", err) + } + + return &ImapClient{client}, nil +} + +// ListMailboxes lists all available mailboxes +func (c *ImapClient) ListMailboxes() ([]string, error) { + var mailboxes []string + cmd := c.List("", "*", nil) + + infos, err := cmd.Collect() + if err != nil { + return nil, err + } + + for _, info := range infos { + mailboxes = append(mailboxes, info.Mailbox) + } + + return mailboxes, nil +} + +// GetMessages retrieves messages from a specific mailbox (simplified version) +func (c *ImapClient) GetMessages(mailbox string, since *time.Time) ([]*Message, error) { + // Select the mailbox + mbox, err := c.Select(mailbox, nil).Wait() + if err != nil { + return nil, fmt.Errorf("failed to select mailbox %s: %w", mailbox, err) + } + + if mbox.NumMessages == 0 { + return []*Message{}, nil + } + + // For now, just return placeholder messages to test the flow + var messages []*Message + numToFetch := mbox.NumMessages + if numToFetch > 5 { + numToFetch = 5 // Limit to 5 messages for testing + } + + for i := uint32(1); i <= numToFetch; i++ { + msg := &Message{ + UID: i, + From: []string{"test@example.com"}, + To: []string{"user@example.com"}, + Subject: fmt.Sprintf("Message %d from %s", i, mailbox), + Date: time.Now(), + Body: fmt.Sprintf("This is a placeholder message %d from mailbox %s", i, mailbox), + Headers: make(map[string][]string), + } + messages = append(messages, msg) + } + + return messages, nil +} + +// ShouldProcessMailbox checks if a mailbox should be processed based on filters +func (c *ImapClient) ShouldProcessMailbox(mailbox string, filter *config.FolderFilter) bool { + // If include list is specified, mailbox must be in it + if len(filter.Include) > 0 { + found := false + for _, included := range filter.Include { + if mailbox == included { + found = true + break + } + } + if !found { + return false + } + } + + // If exclude list is specified, mailbox must not be in it + for _, excluded := range filter.Exclude { + if mailbox == excluded { + return false + } + } + + return true +} + +// Logout logs the client out +func (c *ImapClient) Logout() { + if err := c.Client.Logout(); err != nil { + log.Printf("Failed to logout: %v", err) + } +} diff --git a/go/mail2couch b/go/mail2couch new file mode 100755 index 0000000..2133741 Binary files /dev/null and b/go/mail2couch differ diff --git a/go/main.go b/go/main.go new file mode 100644 index 0000000..485c1fd --- /dev/null +++ b/go/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "mail2couch/config" + "mail2couch/couch" + "mail2couch/mail" +) + +func main() { + cfg, err := config.LoadConfigWithDiscovery() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Initialize CouchDB client + couchClient, err := couch.NewClient(&cfg.CouchDb) + if err != nil { + log.Fatalf("Failed to create CouchDB client: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err = couchClient.EnsureDB(ctx, cfg.CouchDb.Database) + if err != nil { + log.Printf("Could not ensure CouchDB database exists (is it running?): %v", err) + } else { + fmt.Printf("CouchDB database '%s' is ready.\n", cfg.CouchDb.Database) + } + + fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources)) + for _, source := range cfg.MailSources { + if !source.Enabled { + continue + } + + fmt.Printf(" - Processing source: %s\n", source.Name) + if source.Protocol == "imap" { + err := processImapSource(&source, couchClient, cfg.CouchDb.Database) + if err != nil { + log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err) + } + } + } +} + +func processImapSource(source *config.MailSource, couchClient *couch.Client, dbName string) error { + fmt.Printf(" Connecting to IMAP server: %s:%d\n", source.Host, source.Port) + imapClient, err := mail.NewImapClient(source) + if err != nil { + return fmt.Errorf("failed to connect to IMAP server: %w", err) + } + defer imapClient.Logout() + + fmt.Println(" IMAP connection successful.") + + mailboxes, err := imapClient.ListMailboxes() + if err != nil { + return fmt.Errorf("failed to list mailboxes: %w", err) + } + + fmt.Printf(" Found %d mailboxes.\n", len(mailboxes)) + + // Parse the since date if provided + var sinceDate *time.Time + if source.MessageFilter.Since != "" { + parsed, err := time.Parse("2006-01-02", source.MessageFilter.Since) + if err != nil { + log.Printf(" WARNING: Invalid since date format '%s', ignoring filter", source.MessageFilter.Since) + } else { + sinceDate = &parsed + } + } + + totalMessages := 0 + totalStored := 0 + + // Process each mailbox + for _, mailbox := range mailboxes { + // Check if this mailbox should be processed based on filters + if !imapClient.ShouldProcessMailbox(mailbox, &source.FolderFilter) { + fmt.Printf(" Skipping mailbox: %s (filtered)\n", mailbox) + continue + } + + fmt.Printf(" Processing mailbox: %s\n", mailbox) + + // Retrieve messages from the mailbox + messages, err := imapClient.GetMessages(mailbox, sinceDate) + if err != nil { + log.Printf(" ERROR: Failed to get messages from %s: %v", mailbox, err) + continue + } + + if len(messages) == 0 { + fmt.Printf(" No messages found in %s\n", mailbox) + continue + } + + fmt.Printf(" Found %d messages in %s\n", len(messages), mailbox) + totalMessages += len(messages) + + // Convert messages to CouchDB documents + var docs []*couch.MailDocument + for _, msg := range messages { + doc := couch.ConvertMessage(msg, mailbox) + docs = append(docs, doc) + } + + // Store messages in CouchDB + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + stored := 0 + for _, doc := range docs { + err := couchClient.StoreMessage(ctx, dbName, doc) + if err != nil { + log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err) + } else { + stored++ + } + } + cancel() + + fmt.Printf(" Stored %d/%d messages from %s\n", stored, len(messages), mailbox) + totalStored += stored + } + + fmt.Printf(" Summary: Processed %d messages, stored %d new messages\n", totalMessages, totalStored) + return nil +}