olemd/mail2couch
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge pull request 'feature/add-dry-run-mode' (#2) from feature/add-dry-run-mode into main

Browse source 
Reviewed-on: #2
This commit is contained in:
Ole-Morten Duesund 2025-08-03 18:31:16 +02:00
commit 506511a870
6 changed files with 253 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

7
go/config/config.go
View file

@ -86,6 +86,7 @@ func (ms *MailSource) IsArchiveMode() bool {
type CommandLineArgs struct {
ConfigPath string
MaxMessages int
DryRun bool
}
// ParseCommandLine parses command line arguments using GNU-style options
@ -95,6 +96,7 @@ func ParseCommandLine() *CommandLineArgs {
// Define long options with -- and short options with -
pflag.StringVarP(&args.ConfigPath, "config", "c", "", "Path to configuration file")
pflag.IntVarP(&args.MaxMessages, "max-messages", "m", 0, "Maximum number of messages to process per mailbox per run (0 = no limit)")
pflag.BoolVarP(&args.DryRun, "dry-run", "n", false, "Show what would be done without making any changes")
// Add utility options
pflag.BoolP("help", "h", false, "Show help message")
@ -146,7 +148,7 @@ _%s_completions() {
if [[ $cur == -* ]]; then
# Complete with available options
local opts="-c --config -m --max-messages -h --help --generate-bash-completion"
local opts="-c --config -m --max-messages -n --dry-run -h --help --generate-bash-completion"
COMPREPLY=($(compgen -W "$opts" -- "$cur"))
return
fi
@ -217,5 +219,8 @@ func LoadConfigWithDiscovery(args *CommandLineArgs) (*Config, error) {
} else {
fmt.Printf("Maximum messages per mailbox: unlimited\n")
}
if args.DryRun {
fmt.Printf("DRY-RUN MODE: No changes will be made to CouchDB\n")
}
return LoadConfig(configPath)
}

150
go/main.go
View file

@ -19,10 +19,14 @@ func main() {
log.Fatalf("Failed to load configuration: %v", err)
}
// Initialize CouchDB client
couchClient, err := couch.NewClient(&cfg.CouchDb)
if err != nil {
log.Fatalf("Failed to create CouchDB client: %v", err)
// Initialize CouchDB client (skip in dry-run mode)
var couchClient *couch.Client
if !args.DryRun {
var err error
couchClient, err = couch.NewClient(&cfg.CouchDb)
if err != nil {
log.Fatalf("Failed to create CouchDB client: %v", err)
}
}
fmt.Printf("Found %d mail source(s) to process.\n", len(cfg.MailSources))
@ -34,21 +38,25 @@ func main() {
// Generate per-account database name
dbName := couch.GenerateAccountDBName(source.Name, source.User)
// Ensure the account-specific database exists
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err = couchClient.EnsureDB(ctx, dbName)
cancel()
// Ensure the account-specific database exists (skip in dry-run mode)
if !args.DryRun {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
err = couchClient.EnsureDB(ctx, dbName)
cancel()
if err != nil {
log.Printf("Could not ensure CouchDB database '%s' exists (is it running?): %v", dbName, err)
continue
if err != nil {
log.Printf("Could not ensure CouchDB database '%s' exists (is it running?): %v", dbName, err)
continue
} else {
fmt.Printf("CouchDB database '%s' is ready for account: %s\n", dbName, source.Name)
}
} else {
fmt.Printf("CouchDB database '%s' is ready for account: %s\n", dbName, source.Name)
fmt.Printf("DRY-RUN: Would ensure CouchDB database '%s' exists for account: %s\n", dbName, source.Name)
}
fmt.Printf(" - Processing source: %s\n", source.Name)
if source.Protocol == "imap" {
err := processImapSource(&source, couchClient, dbName, args.MaxMessages)
err := processImapSource(&source, couchClient, dbName, args.MaxMessages, args.DryRun)
if err != nil {
log.Printf(" ERROR: Failed to process IMAP source %s: %v", source.Name, err)
}
@ -56,7 +64,7 @@ func main() {
}
}
func processImapSource(source *config.MailSource, couchClient *couch.Client, dbName string, maxMessages int) error {
func processImapSource(source *config.MailSource, couchClient *couch.Client, dbName string, maxMessages int, dryRun bool) error {
fmt.Printf(" Connecting to IMAP server: %s:%d\n", source.Host, source.Port)
imapClient, err := mail.NewImapClient(source)
if err != nil {
@ -92,13 +100,17 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
for _, mailbox := range mailboxes {
fmt.Printf(" Processing mailbox: %s (mode: %s)\n", mailbox, source.Mode)
// Get sync metadata to determine incremental sync date
syncCtx, syncCancel := context.WithTimeout(context.Background(), 10*time.Second)
syncMetadata, err := couchClient.GetSyncMetadata(syncCtx, dbName, mailbox)
syncCancel()
if err != nil {
log.Printf(" ERROR: Failed to get sync metadata for %s: %v", mailbox, err)
continue
// Get sync metadata to determine incremental sync date (skip in dry-run mode)
var syncMetadata *couch.SyncMetadata
if !dryRun {
syncCtx, syncCancel := context.WithTimeout(context.Background(), 10*time.Second)
var err error
syncMetadata, err = couchClient.GetSyncMetadata(syncCtx, dbName, mailbox)
syncCancel()
if err != nil {
log.Printf(" ERROR: Failed to get sync metadata for %s: %v", mailbox, err)
continue
}
}
// Determine the since date for incremental sync
@ -114,7 +126,11 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
if sinceDate != nil {
fmt.Printf(" First sync since: %s (from config)\n", sinceDate.Format("2006-01-02"))
} else {
fmt.Printf(" First full sync (no date filter)\n")
if dryRun {
fmt.Printf(" DRY-RUN: Would perform first full sync (no date filter)\n")
} else {
fmt.Printf(" First full sync (no date filter)\n")
}
}
}
@ -125,13 +141,18 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
continue
}
// Perform sync/archive logic
mailboxSyncCtx, mailboxSyncCancel := context.WithTimeout(context.Background(), 30*time.Second)
err = couchClient.SyncMailbox(mailboxSyncCtx, dbName, mailbox, currentUIDs, source.IsSyncMode())
mailboxSyncCancel()
if err != nil {
log.Printf(" ERROR: Failed to sync mailbox %s: %v", mailbox, err)
continue
// Perform sync/archive logic (skip in dry-run mode)
if !dryRun {
mailboxSyncCtx, mailboxSyncCancel := context.WithTimeout(context.Background(), 30*time.Second)
err = couchClient.SyncMailbox(mailboxSyncCtx, dbName, mailbox, currentUIDs, source.IsSyncMode())
mailboxSyncCancel()
if err != nil {
log.Printf(" ERROR: Failed to sync mailbox %s: %v", mailbox, err)
continue
}
} else {
fmt.Printf(" DRY-RUN: Would sync mailbox %s with %d current UIDs (mode: %s)\n",
mailbox, len(currentUIDs), source.Mode)
}
if len(messages) == 0 {
@ -149,23 +170,32 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
docs = append(docs, doc)
}
// Store messages in CouchDB with attachments
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
// Store messages in CouchDB with attachments (skip in dry-run mode)
stored := 0
for i, doc := range docs {
err := couchClient.StoreMessage(ctx, dbName, doc, messages[i])
if err != nil {
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
} else {
stored++
if !dryRun {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
for i, doc := range docs {
err := couchClient.StoreMessage(ctx, dbName, doc, messages[i])
if err != nil {
log.Printf(" ERROR: Failed to store message %s: %v", doc.ID, err)
} else {
stored++
}
}
cancel()
fmt.Printf(" Stored %d/%d messages from %s\n", stored, len(messages), mailbox)
} else {
stored = len(messages) // In dry-run, assume all would be stored
fmt.Printf(" DRY-RUN: Would store %d messages from %s\n", len(messages), mailbox)
// Show sample of what would be stored
if len(docs) > 0 {
fmt.Printf(" DRY-RUN: Sample message ID: %s (Subject: %s)\n",
docs[0].ID, docs[0].Subject)
}
}
cancel()
fmt.Printf(" Stored %d/%d messages from %s\n", stored, len(messages), mailbox)
totalStored += stored
// Update sync metadata after successful processing
// Update sync metadata after successful processing (skip in dry-run mode)
if len(messages) > 0 {
// Find the highest UID processed
var maxUID uint32
@ -175,26 +205,34 @@ func processImapSource(source *config.MailSource, couchClient *couch.Client, dbN
}
}
// Create/update sync metadata
newMetadata := &couch.SyncMetadata{
Mailbox: mailbox,
LastSyncTime: time.Now(),
LastMessageUID: maxUID,
MessageCount: stored,
}
if !dryRun {
// Create/update sync metadata
newMetadata := &couch.SyncMetadata{
Mailbox: mailbox,
LastSyncTime: time.Now(),
LastMessageUID: maxUID,
MessageCount: stored,
}
// Store sync metadata
metadataCtx, metadataCancel := context.WithTimeout(context.Background(), 10*time.Second)
err = couchClient.StoreSyncMetadata(metadataCtx, dbName, newMetadata)
metadataCancel()
if err != nil {
log.Printf(" WARNING: Failed to store sync metadata for %s: %v", mailbox, err)
// Store sync metadata
metadataCtx, metadataCancel := context.WithTimeout(context.Background(), 10*time.Second)
err = couchClient.StoreSyncMetadata(metadataCtx, dbName, newMetadata)
metadataCancel()
if err != nil {
log.Printf(" WARNING: Failed to store sync metadata for %s: %v", mailbox, err)
} else {
fmt.Printf(" Updated sync metadata (last UID: %d)\n", maxUID)
}
} else {
fmt.Printf(" Updated sync metadata (last UID: %d)\n", maxUID)
fmt.Printf(" DRY-RUN: Would update sync metadata (last UID: %d, %d messages)\n", maxUID, stored)
}
}
}
fmt.Printf(" Summary: Processed %d messages, stored %d new messages\n", totalMessages, totalStored)
if dryRun {
fmt.Printf(" DRY-RUN Summary: Found %d messages, would store %d messages\n", totalMessages, totalStored)
} else {
fmt.Printf(" Summary: Processed %d messages, stored %d new messages\n", totalMessages, totalStored)
}
return nil
}

8
rust/src/cli.rs
View file

@ -28,6 +28,11 @@ pub fn parse_command_line() -> CommandLineArgs {
.help("Maximum number of messages to process per mailbox per run (0 = no limit)")
.value_parser(clap::value_parser!(u32))
.action(ArgAction::Set))
.arg(Arg::new("dry-run")
.short('n')
.long("dry-run")
.help("Show what would be done without making any changes")
.action(ArgAction::SetTrue))
.arg(Arg::new("generate-bash-completion")
.long("generate-bash-completion")
.help("Generate bash completion script and exit")
@ -44,6 +49,7 @@ pub fn parse_command_line() -> CommandLineArgs {
CommandLineArgs {
config_path: matches.get_one::<String>("config").map(|s| s.clone()),
max_messages: matches.get_one::<u32>("max-messages").copied(),
dry_run: matches.get_flag("dry-run"),
generate_bash_completion: matches.get_flag("generate-bash-completion"),
help: false, // Using clap's built-in help
}
@ -83,7 +89,7 @@ _{}_completions() {{
if [[ $cur == -* ]]; then
# Complete with available options
local opts="-c --config -m --max-messages -h --help --generate-bash-completion"
local opts="-c --config -m --max-messages -n --dry-run -h --help --generate-bash-completion"
COMPREPLY=($(compgen -W "$opts" -- "$cur"))
return
fi

2
rust/src/config.rs
View file

@ -107,6 +107,7 @@ impl MailSource {
pub struct CommandLineArgs {
pub config_path: Option<String>,
pub max_messages: Option<u32>,
pub dry_run: bool,
pub generate_bash_completion: bool,
pub help: bool,
}
@ -283,6 +284,7 @@ mod tests {
let args = CommandLineArgs {
config_path: None,
max_messages: None,
dry_run: false,
generate_bash_completion: false,
help: false,
};

4
rust/src/main.rs
View file

@ -36,6 +36,10 @@ async fn run(args: mail2couch::config::CommandLineArgs) -> Result<()> {
info!("Maximum messages per mailbox: unlimited");
}
if args.dry_run {
info!("🔍 DRY-RUN MODE: No changes will be made to CouchDB");
}
// Display configuration summary
print_config_summary(&config);

209
rust/src/sync.rs
View file

@ -119,8 +119,12 @@ impl SyncCoordinator {
let db_name = generate_database_name(&source.name, &source.user);
info!("Using database: {}", db_name);
// Create database if it doesn't exist
self.couch_client.create_database(&db_name).await?;
// Create database if it doesn't exist (skip in dry-run mode)
if !self.args.dry_run {
self.couch_client.create_database(&db_name).await?;
} else {
info!("🔍 DRY-RUN: Would create database {}", db_name);
}
// Connect to IMAP server
let mut imap_client = ImapClient::connect(source.clone()).await?;
@ -212,22 +216,33 @@ impl SyncCoordinator {
let mailbox_info = imap_client.select_mailbox(mailbox).await?;
debug!("Selected mailbox {}: {} messages", mailbox, mailbox_info.exists);
// Get last sync metadata
let since_date = match self.couch_client.get_sync_metadata(db_name, mailbox).await {
Ok(metadata) => {
info!(" Found sync metadata, last sync: {}", metadata.last_sync_time);
Some(metadata.last_sync_time)
}
Err(_) => {
info!(" No sync metadata found, performing full sync");
// Parse since date from message filter if provided
source.message_filter.since.as_ref()
.and_then(|since_str| {
DateTime::parse_from_str(&format!("{} 00:00:00 +0000", since_str), "%Y-%m-%d %H:%M:%S %z")
.map(|dt| dt.with_timezone(&Utc))
.ok()
})
// Get last sync metadata (skip in dry-run mode)
let since_date = if !self.args.dry_run {
match self.couch_client.get_sync_metadata(db_name, mailbox).await {
Ok(metadata) => {
info!(" Found sync metadata, last sync: {}", metadata.last_sync_time);
Some(metadata.last_sync_time)
}
Err(_) => {
info!(" No sync metadata found, performing full sync");
// Parse since date from message filter if provided
source.message_filter.since.as_ref()
.and_then(|since_str| {
DateTime::parse_from_str(&format!("{} 00:00:00 +0000", since_str), "%Y-%m-%d %H:%M:%S %z")
.map(|dt| dt.with_timezone(&Utc))
.ok()
})
}
}
} else {
info!(" 🔍 DRY-RUN: Would check for sync metadata");
// In dry-run mode, use config since date if available
source.message_filter.since.as_ref()
.and_then(|since_str| {
DateTime::parse_from_str(&format!("{} 00:00:00 +0000", since_str), "%Y-%m-%d %H:%M:%S %z")
.map(|dt| dt.with_timezone(&Utc))
.ok()
})
};
// Search for messages using server-side IMAP SEARCH with keyword filtering when possible
@ -257,17 +272,21 @@ impl SyncCoordinator {
};
info!(" Found {} messages matching search criteria", message_uids.len());
// Handle sync mode - check for deleted messages
// Handle sync mode - check for deleted messages (skip in dry-run mode)
let mut messages_deleted = 0;
if source.mode == "sync" {
messages_deleted = self.handle_deleted_messages(db_name, mailbox, &message_uids).await
.unwrap_or_else(|e| {
warn!(" Failed to handle deleted messages: {}", e);
0
});
if !self.args.dry_run {
messages_deleted = self.handle_deleted_messages(db_name, mailbox, &message_uids).await
.unwrap_or_else(|e| {
warn!(" Failed to handle deleted messages: {}", e);
0
});
if messages_deleted > 0 {
info!(" 🗑️ Deleted {} messages that no longer exist on server", messages_deleted);
if messages_deleted > 0 {
info!(" 🗑️ Deleted {} messages that no longer exist on server", messages_deleted);
}
} else {
info!(" 🔍 DRY-RUN: Would check for deleted messages in sync mode");
}
}
@ -312,54 +331,74 @@ impl SyncCoordinator {
// Extract UID before moving the document
let uid_str = mail_doc.source_uid.clone();
// Store the message document first
match self.couch_client.store_mail_document(db_name, mail_doc).await {
Ok(doc_id) => {
messages_stored += 1;
// Store the message document first (skip in dry-run mode)
if !self.args.dry_run {
match self.couch_client.store_mail_document(db_name, mail_doc).await {
Ok(doc_id) => {
messages_stored += 1;
// Store attachments if any exist
if !attachments.is_empty() {
for (filename, content_type, data) in attachments {
match self.couch_client.store_attachment(
db_name,
&doc_id,
&filename,
&content_type,
&data,
).await {
Ok(_) => {
debug!(" Stored attachment: {}", filename);
}
Err(e) => {
warn!(" Failed to store attachment {}: {}", filename, e);
// Store attachments if any exist
if !attachments.is_empty() {
for (filename, content_type, data) in attachments {
match self.couch_client.store_attachment(
db_name,
&doc_id,
&filename,
&content_type,
&data,
).await {
Ok(_) => {
debug!(" Stored attachment: {}", filename);
}
Err(e) => {
warn!(" Failed to store attachment {}: {}", filename, e);
}
}
}
}
}
// Parse UID from source_uid
if let Ok(uid) = uid_str.parse::<u32>() {
last_uid = Some(last_uid.map_or(uid, |prev: u32| prev.max(uid)));
// Parse UID from source_uid
if let Ok(uid) = uid_str.parse::<u32>() {
last_uid = Some(last_uid.map_or(uid, |prev: u32| prev.max(uid)));
}
}
Err(e) => {
warn!(" Failed to store message {}: {}", uid_str, e);
messages_skipped += 1;
}
}
Err(e) => {
warn!(" Failed to store message {}: {}", uid_str, e);
messages_skipped += 1;
} else {
// In dry-run mode, simulate successful storage
messages_stored += 1;
debug!(" 🔍 DRY-RUN: Would store message {} (Subject: {})",
uid_str, mail_doc.subject);
if !attachments.is_empty() {
debug!(" 🔍 DRY-RUN: Would store {} attachments", attachments.len());
}
// Parse UID from source_uid
if let Ok(uid) = uid_str.parse::<u32>() {
last_uid = Some(last_uid.map_or(uid, |prev: u32| prev.max(uid)));
}
}
}
// Update sync metadata
// Update sync metadata (skip in dry-run mode)
if let Some(uid) = last_uid {
let sync_metadata = SyncMetadata::new(
mailbox.to_string(),
start_time,
uid,
messages_stored,
);
if !self.args.dry_run {
let sync_metadata = SyncMetadata::new(
mailbox.to_string(),
start_time,
uid,
messages_stored,
);
if let Err(e) = self.couch_client.store_sync_metadata(db_name, &sync_metadata).await {
warn!(" Failed to store sync metadata: {}", e);
if let Err(e) = self.couch_client.store_sync_metadata(db_name, &sync_metadata).await {
warn!(" Failed to store sync metadata: {}", e);
}
} else {
info!(" 🔍 DRY-RUN: Would update sync metadata (last UID: {}, {} messages)", uid, messages_stored);
}
}
@ -422,7 +461,11 @@ impl SyncCoordinator {
/// Print summary of sync results
pub fn print_sync_summary(&self, results: &[SourceSyncResult]) {
info!("\n🎉 Synchronization completed!");
if self.args.dry_run {
info!("\n🔍 DRY-RUN completed!");
} else {
info!("\n🎉 Synchronization completed!");
}
info!("{}", "=".repeat(50));
let mut total_sources = 0;
@ -434,24 +477,45 @@ impl SyncCoordinator {
total_mailboxes += result.mailboxes_processed;
total_messages += result.total_messages;
info!(
"📧 {}: {} mailboxes, {} messages (database: {})",
result.source_name,
result.mailboxes_processed,
result.total_messages,
result.database
);
if self.args.dry_run {
info!(
"📧 {}: {} mailboxes, {} messages found (database: {})",
result.source_name,
result.mailboxes_processed,
result.total_messages,
result.database
);
} else {
info!(
"📧 {}: {} mailboxes, {} messages (database: {})",
result.source_name,
result.mailboxes_processed,
result.total_messages,
result.database
);
}
}
info!("{}", "=".repeat(50));
info!(
"📊 Total: {} sources, {} mailboxes, {} messages",
total_sources, total_mailboxes, total_messages
);
if self.args.dry_run {
info!(
"📊 DRY-RUN Total: {} sources, {} mailboxes, {} messages found",
total_sources, total_mailboxes, total_messages
);
} else {
info!(
"📊 Total: {} sources, {} mailboxes, {} messages",
total_sources, total_mailboxes, total_messages
);
}
if let Some(max) = self.args.max_messages {
info!("⚠️ Message limit was applied: {} per mailbox", max);
}
if self.args.dry_run {
info!("🔍 No changes were made to CouchDB");
}
}
}
@ -493,6 +557,7 @@ mod tests {
let args = CommandLineArgs {
config_path: None,
max_messages: Some(10),
dry_run: false,
generate_bash_completion: false,
help: false,
};