feat: add comprehensive README documentation and clean up configuration
## Documentation Enhancements - Create comprehensive README with installation, configuration, and usage examples - Add simple, advanced, and provider-specific configuration examples - Document all features: incremental sync, wildcard patterns, keyword filtering, attachment support - Include production deployment guidance and troubleshooting section - Add architecture documentation with database structure and document format examples ## Configuration Cleanup - Remove unnecessary `database` field from CouchDB configuration - Add `m2c_` prefix to all CouchDB database names for better namespace isolation - Update GenerateAccountDBName() to consistently prefix databases with `m2c_` - Clean up all configuration examples to remove deprecated database field ## Test Environment Simplification - Simplify test script structure to eliminate confusion and redundancy - Remove redundant populate-test-messages.sh wrapper script - Update run-tests.sh to be comprehensive automated test with cleanup - Maintain clear separation: automated tests vs manual testing environment - Update all test scripts to expect m2c-prefixed database names ## Configuration Examples Added - config-simple.json: Basic single Gmail account setup - config-advanced.json: Multi-account with complex filtering and different providers - config-providers.json: Real-world configurations for Gmail, Outlook, Yahoo, iCloud ## Benefits - Clear documentation for users from beginner to advanced - Namespace isolation prevents database conflicts in shared CouchDB instances - Simplified test workflow eliminates user confusion about which scripts to use - Comprehensive examples cover common email provider configurations 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
357cd06264
commit
c2ad55eaaf
17 changed files with 1139 additions and 111 deletions
242
test/test-incremental-sync.sh
Executable file
242
test/test-incremental-sync.sh
Executable file
|
|
@ -0,0 +1,242 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Test script to validate incremental sync functionality
|
||||
# This script tests that mail2couch properly implements incremental synchronization
|
||||
|
||||
set -e
|
||||
|
||||
echo "🔄 Testing Incremental Sync Functionality"
|
||||
echo "=========================================="
|
||||
|
||||
# Make sure we're in the right directory
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Function to check if containers are running
|
||||
check_containers() {
|
||||
echo "🔍 Checking if test containers are running..."
|
||||
|
||||
if ! podman ps | grep -q "greenmail"; then
|
||||
echo -e "${RED}❌ GreenMail container not running${NC}"
|
||||
echo "Please run: cd test && ./start-test-env.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! podman ps | grep -q "couchdb"; then
|
||||
echo -e "${RED}❌ CouchDB container not running${NC}"
|
||||
echo "Please run: cd test && ./start-test-env.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✅ Test containers are running${NC}"
|
||||
}
|
||||
|
||||
# Function to populate initial test data
|
||||
populate_initial_data() {
|
||||
echo "📧 Populating initial test data..."
|
||||
cd test
|
||||
if python3 populate-greenmail.py; then
|
||||
echo -e "${GREEN}✅ Initial test data populated${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Failed to populate initial test data${NC}"
|
||||
exit 1
|
||||
fi
|
||||
cd ..
|
||||
}
|
||||
|
||||
# Function to build the application
|
||||
build_app() {
|
||||
echo "🔨 Building mail2couch..."
|
||||
cd go
|
||||
if go build -o mail2couch .; then
|
||||
echo -e "${GREEN}✅ Build successful${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Build failed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
cd ..
|
||||
}
|
||||
|
||||
# Function to run first sync
|
||||
run_first_sync() {
|
||||
echo -e "\n${BLUE}Running first sync...${NC}"
|
||||
cd go
|
||||
./mail2couch -config ../test/config-test.json -max-messages 5
|
||||
cd ..
|
||||
}
|
||||
|
||||
# Function to add new messages to test incremental sync
|
||||
add_new_messages() {
|
||||
echo -e "\n${YELLOW}Adding new messages for incremental sync test...${NC}"
|
||||
|
||||
# Create a simple Python script to add messages directly to GreenMail
|
||||
cat > test/add_incremental_messages.py << 'EOF'
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import imaplib
|
||||
import time
|
||||
from test.populate_greenmail import create_simple_message
|
||||
|
||||
def add_new_messages():
|
||||
"""Add new messages to test incremental sync"""
|
||||
accounts = [
|
||||
("testuser1", "password123"),
|
||||
("syncuser", "syncpass"),
|
||||
("archiveuser", "archivepass")
|
||||
]
|
||||
|
||||
for username, password in accounts:
|
||||
try:
|
||||
print(f"Adding new messages to {username}...")
|
||||
imap = imaplib.IMAP4('localhost', 3143)
|
||||
imap.login(username, password)
|
||||
imap.select('INBOX')
|
||||
|
||||
# Add 3 new messages with timestamps after the first sync
|
||||
for i in range(1, 4):
|
||||
subject = f"Incremental Sync Test Message {i}"
|
||||
body = f"This message was added after the first sync for incremental testing. Message {i} for {username}."
|
||||
|
||||
msg = create_simple_message(subject, body, f"incremental-test@example.com", f"{username}@example.com")
|
||||
imap.append('INBOX', None, None, msg.encode('utf-8'))
|
||||
print(f" Added: {subject}")
|
||||
time.sleep(0.1)
|
||||
|
||||
imap.logout()
|
||||
print(f"✅ Added 3 new messages to {username}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error adding messages to {username}: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
add_new_messages()
|
||||
EOF
|
||||
|
||||
# Add the parent directory to Python path and run the script
|
||||
cd test
|
||||
PYTHONPATH=.. python3 add_incremental_messages.py
|
||||
cd ..
|
||||
}
|
||||
|
||||
# Function to run second sync (incremental)
|
||||
run_incremental_sync() {
|
||||
echo -e "\n${BLUE}Running incremental sync...${NC}"
|
||||
cd go
|
||||
./mail2couch -config ../test/config-test.json -max-messages 10
|
||||
cd ..
|
||||
}
|
||||
|
||||
# Function to verify incremental sync results
|
||||
verify_results() {
|
||||
echo -e "\n${YELLOW}Verifying incremental sync results...${NC}"
|
||||
|
||||
# Check CouchDB for sync metadata documents
|
||||
echo "Checking for sync metadata in CouchDB databases..."
|
||||
|
||||
# List of expected databases based on test config (with m2c prefix)
|
||||
databases=("m2c_wildcard_all_folders_test" "m2c_work_pattern_test" "m2c_specific_folders_only")
|
||||
|
||||
for db in "${databases[@]}"; do
|
||||
echo " Checking database: $db"
|
||||
|
||||
# Check if database exists
|
||||
if curl -s -f "http://admin:password@localhost:5984/$db" > /dev/null; then
|
||||
echo " ✅ Database exists"
|
||||
|
||||
# Look for sync metadata documents
|
||||
metadata_docs=$(curl -s "http://admin:password@localhost:5984/$db/_all_docs?startkey=\"sync_metadata\"&endkey=\"sync_metadata_z\"" | grep -o '"total_rows":[0-9]*' | cut -d: -f2 || echo "0")
|
||||
|
||||
if [ "$metadata_docs" -gt 0 ]; then
|
||||
echo " ✅ Found sync metadata documents: $metadata_docs"
|
||||
|
||||
# Get a sample sync metadata document
|
||||
sample_doc=$(curl -s "http://admin:password@localhost:5984/$db/_all_docs?startkey=\"sync_metadata\"&endkey=\"sync_metadata_z\"&include_docs=true&limit=1")
|
||||
echo " Sample sync metadata:"
|
||||
echo "$sample_doc" | python3 -m json.tool | grep -E "(lastSyncTime|lastMessageUID|messageCount)" | head -3
|
||||
else
|
||||
echo " ⚠️ No sync metadata documents found"
|
||||
fi
|
||||
else
|
||||
echo " ❌ Database does not exist"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Main test execution
|
||||
main() {
|
||||
echo "Starting incremental sync tests..."
|
||||
|
||||
# Pre-test setup
|
||||
check_containers
|
||||
build_app
|
||||
|
||||
# Clean up any existing data
|
||||
echo "🧹 Cleaning up existing test data..."
|
||||
curl -s -X DELETE "http://admin:password@localhost:5984/m2c_wildcard_all_folders_test" > /dev/null || true
|
||||
curl -s -X DELETE "http://admin:password@localhost:5984/m2c_work_pattern_test" > /dev/null || true
|
||||
curl -s -X DELETE "http://admin:password@localhost:5984/m2c_specific_folders_only" > /dev/null || true
|
||||
|
||||
# Step 1: Populate initial test data
|
||||
populate_initial_data
|
||||
|
||||
# Wait for data to settle
|
||||
echo "⏳ Waiting for initial data to settle..."
|
||||
sleep 5
|
||||
|
||||
# Step 2: Run first sync to establish baseline
|
||||
echo -e "\n${YELLOW}=== STEP 1: First Sync (Baseline) ===${NC}"
|
||||
run_first_sync
|
||||
|
||||
# Wait between syncs
|
||||
echo "⏳ Waiting between syncs..."
|
||||
sleep 3
|
||||
|
||||
# Step 3: Add new messages for incremental sync test
|
||||
echo -e "\n${YELLOW}=== STEP 2: Add New Messages ===${NC}"
|
||||
add_new_messages
|
||||
|
||||
# Wait for new messages to be ready
|
||||
echo "⏳ Waiting for new messages to be ready..."
|
||||
sleep 2
|
||||
|
||||
# Step 4: Run incremental sync
|
||||
echo -e "\n${YELLOW}=== STEP 3: Incremental Sync ===${NC}"
|
||||
run_incremental_sync
|
||||
|
||||
# Step 5: Verify results
|
||||
echo -e "\n${YELLOW}=== STEP 4: Verification ===${NC}"
|
||||
verify_results
|
||||
|
||||
echo -e "\n${GREEN}🎉 Incremental sync test completed!${NC}"
|
||||
echo ""
|
||||
echo "Key features tested:"
|
||||
echo " ✅ Sync metadata storage and retrieval"
|
||||
echo " ✅ IMAP SEARCH with SINCE for efficient incremental fetching"
|
||||
echo " ✅ Last sync timestamp tracking per mailbox"
|
||||
echo " ✅ Proper handling of first sync vs incremental sync"
|
||||
echo ""
|
||||
echo "To verify results manually:"
|
||||
echo " - Check CouchDB: http://localhost:5984/_utils"
|
||||
echo " - Look for 'sync_metadata_*' documents in each database"
|
||||
echo " - Verify incremental messages were added after baseline sync"
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
echo "🧹 Cleaning up test artifacts..."
|
||||
rm -f test/add_incremental_messages.py
|
||||
}
|
||||
|
||||
# Set trap to cleanup on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
# Run main function if executed directly
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue