skyview/cmd/skyview-data/main.go

// Package main implements the SkyView data management utility.
//
// This tool provides simple commands for populating and updating the SkyView
// database with aviation data from various external sources while maintaining
// proper license compliance.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"skyview/internal/database"
)

// Shared configuration structures (should match main skyview)
type Config struct {
	Server   ServerConfig          `json:"server"`             
	Sources  []SourceConfig        `json:"sources"`            
	Settings Settings              `json:"settings"`           
	Database *database.Config      `json:"database,omitempty"` 
	Callsign *CallsignConfig       `json:"callsign,omitempty"` 
	Origin   OriginConfig          `json:"origin"`             
}

type CallsignConfig struct {
	Enabled     bool                              `json:"enabled"`
	CacheHours  int                               `json:"cache_hours"`
	PrivacyMode bool                              `json:"privacy_mode"`
	Sources     map[string]CallsignSourceConfig   `json:"sources"`
	ExternalAPIs map[string]ExternalAPIConfig     `json:"external_apis,omitempty"`
}

type CallsignSourceConfig struct {
	Enabled         bool   `json:"enabled"`
	Priority        int    `json:"priority"`
	License         string `json:"license"`
	RequiresConsent bool   `json:"requires_consent,omitempty"`
	UserAcceptsTerms bool  `json:"user_accepts_terms,omitempty"`
}

type ExternalAPIConfig struct {
	Enabled         bool `json:"enabled"`
	TimeoutSeconds  int  `json:"timeout_seconds,omitempty"`
	MaxRetries      int  `json:"max_retries,omitempty"`
	RequiresConsent bool `json:"requires_consent,omitempty"`
}

type OriginConfig struct {
	Latitude  float64 `json:"latitude"`       
	Longitude float64 `json:"longitude"`      
	Name      string  `json:"name,omitempty"` 
}

type ServerConfig struct {
	Host string `json:"host"` 
	Port int    `json:"port"` 
}

type SourceConfig struct {
	ID        string  `json:"id"`
	Name      string  `json:"name"`
	Host      string  `json:"host"`
	Port      int     `json:"port"`
	Format    string  `json:"format,omitempty"`
	Latitude  float64 `json:"latitude"`
	Longitude float64 `json:"longitude"`
	Altitude  float64 `json:"altitude"`
	Enabled   bool    `json:"enabled"`
}

type Settings struct {
	HistoryLimit int `json:"history_limit"`
	StaleTimeout int `json:"stale_timeout"`
	UpdateRate   int `json:"update_rate"`
}

// Version information
var (
	version = "dev"
	commit  = "unknown"
	date    = "unknown"
)

func main() {
	var (
		configPath = flag.String("config", "config.json", "Configuration file path")
		dbPath     = flag.String("db", "", "Database file path (override config)")
		verbose    = flag.Bool("v", false, "Verbose output")
		force      = flag.Bool("force", false, "Force operation without prompts")
		showVer    = flag.Bool("version", false, "Show version information")
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `SkyView Data Manager v%s

USAGE:
    skyview-data [OPTIONS] COMMAND [ARGS...]

COMMANDS:
    init                    Initialize empty database
    list                    List available data sources  
    status                  Show current database status
    update [SOURCE...]      Update data from sources (default: safe sources)
    import SOURCE           Import data from specific source
    clear SOURCE            Remove data from specific source
    reset                   Clear all data and reset database
    optimize                Optimize database for storage efficiency

EXAMPLES:
    skyview-data init                    # Create empty database
    skyview-data update                  # Update from safe (public domain) sources
    skyview-data update openflights     # Update OpenFlights data (requires license acceptance)
    skyview-data import ourairports     # Import OurAirports data  
    skyview-data list                    # Show available sources
    skyview-data status                  # Show database status
    skyview-data optimize                # Optimize database storage

OPTIONS:
`, version)
		flag.PrintDefaults()
	}

	flag.Parse()

	if *showVer {
		fmt.Printf("skyview-data version %s (commit %s, built %s)\n", version, commit, date)
		return
	}

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	command := flag.Arg(0)

	// Set up logging for cron-friendly operation
	if *verbose {
		log.SetFlags(log.LstdFlags | log.Lshortfile)
	} else {
		// For cron jobs, include timestamp but no file info
		log.SetFlags(log.LstdFlags)
	}

	// Load configuration
	config, err := loadConfig(*configPath)
	if err != nil {
		log.Fatalf("Configuration loading failed: %v", err)
	}
	
	// Initialize database connection using shared config
	db, err := initDatabaseFromConfig(config, *dbPath)
	if err != nil {
		log.Fatalf("Database initialization failed: %v", err)
	}
	defer db.Close()

	// Execute command
	switch command {
	case "init":
		err = cmdInit(db, *force)
	case "list":
		err = cmdList(db)
	case "status":
		err = cmdStatus(db)
	case "update":
		sources := flag.Args()[1:]
		err = cmdUpdate(db, sources, *force)
	case "import":
		if flag.NArg() < 2 {
			log.Fatal("import command requires a source name")
		}
		err = cmdImport(db, flag.Arg(1), *force)
	case "clear":
		if flag.NArg() < 2 {
			log.Fatal("clear command requires a source name")
		}
		err = cmdClear(db, flag.Arg(1), *force)
	case "reset":
		err = cmdReset(db, *force)
	case "optimize":
		err = cmdOptimize(db, *force)
	default:
		log.Fatalf("Unknown command: %s", command)
	}

	if err != nil {
		log.Fatalf("Command failed: %v", err)
	}
}

// initDatabase initializes the database connection with auto-creation
func initDatabase(dbPath string) (*database.Database, error) {
	config := database.DefaultConfig()
	config.Path = dbPath

	db, err := database.NewDatabase(config)
	if err != nil {
		return nil, fmt.Errorf("failed to create database: %v", err)
	}

	if err := db.Initialize(); err != nil {
		db.Close()
		return nil, fmt.Errorf("failed to initialize database: %v", err)
	}

	return db, nil
}

// cmdInit initializes an empty database
func cmdInit(db *database.Database, force bool) error {
	dbPath := db.GetConfig().Path
	
	// Check if database already exists and has data
	if !force {
		if stats, err := db.GetHistoryManager().GetStatistics(); err == nil {
			if totalRecords, ok := stats["total_records"].(int); ok && totalRecords > 0 {
				fmt.Printf("Database already exists with %d records at: %s\n", totalRecords, dbPath)
				fmt.Println("Use --force to reinitialize")
				return nil
			}
		}
	}

	fmt.Printf("Initializing SkyView database at: %s\n", dbPath)
	fmt.Println("✓ Database schema created")
	fmt.Println("✓ Empty tables ready for data import")
	fmt.Println()
	fmt.Println("Next steps:")
	fmt.Println("  skyview-data update              # Import safe (public domain) data")
	fmt.Println("  skyview-data list                # Show available data sources")
	fmt.Println("  skyview-data status              # Check database status")

	return nil
}

// cmdList shows available data sources
func cmdList(db *database.Database) error {
	fmt.Println("Available Data Sources:")
	fmt.Println()

	sources := database.GetAvailableDataSources()
	for _, source := range sources {
		status := "🟢"
		if source.RequiresConsent {
			status = "⚠️ "
		}

		fmt.Printf("%s %s\n", status, source.Name)
		fmt.Printf("   License: %s\n", source.License)
		fmt.Printf("   URL: %s\n", source.URL)
		if source.RequiresConsent {
			fmt.Printf("   Note: Requires license acceptance\n")
		}
		fmt.Println()
	}

	fmt.Println("Legend:")
	fmt.Println("  🟢 = Safe to use automatically (Public Domain/MIT)")
	fmt.Println("  ⚠️  = Requires license acceptance (AGPL, etc.)")

	return nil
}

// cmdStatus shows current database status
func cmdStatus(db *database.Database) error {
	fmt.Println("SkyView Database Status")
	fmt.Println("======================")
	
	dbPath := db.GetConfig().Path
	fmt.Printf("Database: %s\n", dbPath)
	
	// Check if file exists and get size
	if stat, err := os.Stat(dbPath); err == nil {
		fmt.Printf("Size: %.2f MB\n", float64(stat.Size())/(1024*1024))
		fmt.Printf("Modified: %s\n", stat.ModTime().Format(time.RFC3339))
		
		// Add database optimization stats
		optimizer := database.NewOptimizationManager(db, db.GetConfig())
		if stats, err := optimizer.GetOptimizationStats(); err == nil {
			fmt.Printf("Efficiency: %.1f%% (%d used pages, %d free pages)\n", 
				stats.Efficiency, stats.UsedPages, stats.FreePages)
			if stats.AutoVacuumEnabled {
				fmt.Printf("Auto-VACUUM: Enabled\n")
			}
		}
	}
	fmt.Println()

	// Show loaded data sources
	loader := database.NewDataLoader(db.GetConnection())
	loadedSources, err := loader.GetLoadedDataSources()
	if err != nil {
		return fmt.Errorf("failed to get loaded sources: %v", err)
	}

	if len(loadedSources) == 0 {
		fmt.Println("📭 No data sources loaded")
		fmt.Println("   Run 'skyview-data update' to populate with aviation data")
	} else {
		fmt.Printf("📦 Loaded Data Sources (%d):\n", len(loadedSources))
		for _, source := range loadedSources {
			fmt.Printf("   • %s (%s)\n", source.Name, source.License)
		}
	}
	fmt.Println()

	// Show statistics
	stats, err := db.GetHistoryManager().GetStatistics()
	if err != nil {
		return fmt.Errorf("failed to get statistics: %v", err)
	}

	// Show comprehensive reference data statistics
	var airportCount, airlineCount int
	db.GetConnection().QueryRow(`SELECT COUNT(*) FROM airports`).Scan(&airportCount)
	db.GetConnection().QueryRow(`SELECT COUNT(*) FROM airlines`).Scan(&airlineCount)
	
	// Get data source update information
	var lastUpdate time.Time
	var updateCount int
	err = db.GetConnection().QueryRow(`
		SELECT COUNT(*), MAX(imported_at) 
		FROM data_sources 
		WHERE imported_at IS NOT NULL
	`).Scan(&updateCount, &lastUpdate)
	
	fmt.Printf("📊 Database Statistics:\n")
	fmt.Printf("   Reference Data:\n")
	if airportCount > 0 {
		fmt.Printf("     • Airports: %d\n", airportCount)
	}
	if airlineCount > 0 {
		fmt.Printf("     • Airlines: %d\n", airlineCount)
	}
	if updateCount > 0 {
		fmt.Printf("     • Data Sources: %d imported\n", updateCount)
		if !lastUpdate.IsZero() {
			fmt.Printf("     • Last Updated: %s\n", lastUpdate.Format("2006-01-02 15:04:05"))
		}
	}
	
	fmt.Printf("   Flight History:\n")
	if totalRecords, ok := stats["total_records"].(int); ok {
		fmt.Printf("     • Aircraft Records: %d\n", totalRecords)
	}
	if uniqueAircraft, ok := stats["unique_aircraft"].(int); ok {
		fmt.Printf("     • Unique Aircraft: %d\n", uniqueAircraft)
	}
	if recentRecords, ok := stats["recent_records_24h"].(int); ok {
		fmt.Printf("     • Last 24h: %d records\n", recentRecords)
	}

	oldestRecord, hasOldest := stats["oldest_record"]
	newestRecord, hasNewest := stats["newest_record"]
	if hasOldest && hasNewest && oldestRecord != nil && newestRecord != nil {
		if oldest, ok := oldestRecord.(time.Time); ok {
			if newest, ok := newestRecord.(time.Time); ok {
				fmt.Printf("     • Flight Data Range: %s to %s\n", 
					oldest.Format("2006-01-02"),
					newest.Format("2006-01-02"))
			}
		}
	}
	
	// Show airport data sample if available
	if airportCount > 0 {
		var sampleAirports []string
		rows, err := db.GetConnection().Query(`
			SELECT name || ' (' || COALESCE(icao_code, ident) || ')' 
			FROM airports 
			WHERE icao_code IS NOT NULL AND icao_code != ''
			ORDER BY name 
			LIMIT 3
		`)
		if err == nil {
			for rows.Next() {
				var airport string
				if rows.Scan(&airport) == nil {
					sampleAirports = append(sampleAirports, airport)
				}
			}
			rows.Close()
			if len(sampleAirports) > 0 {
				fmt.Printf("     • Sample Airports: %s\n", strings.Join(sampleAirports, ", "))
			}
		}
	}

	return nil
}

// cmdUpdate updates data from specified sources (or safe sources by default)
func cmdUpdate(db *database.Database, sources []string, force bool) error {
	availableSources := database.GetAvailableDataSources()
	
	// If no sources specified, use safe (non-consent-required) sources
	if len(sources) == 0 {
		log.Println("Updating from safe data sources...")
		for _, source := range availableSources {
			if !source.RequiresConsent {
				sources = append(sources, strings.ToLower(strings.ReplaceAll(source.Name, " ", "")))
			}
		}
		
		if len(sources) == 0 {
			log.Println("No safe data sources available for automatic update")
			return nil
		}
		log.Printf("Found %d safe data sources to update", len(sources))
	}

	loader := database.NewDataLoader(db.GetConnection())
	
	for _, sourceName := range sources {
		// Find matching source
		var matchedSource *database.DataSource
		for _, available := range availableSources {
			if strings.EqualFold(strings.ReplaceAll(available.Name, " ", ""), sourceName) {
				matchedSource = &available
				break
			}
		}
		
		if matchedSource == nil {
			log.Printf("⚠️  Unknown source: %s", sourceName)
			continue
		}

		// Check for consent requirement  
		if matchedSource.RequiresConsent && !force {
			log.Printf("Skipping %s: requires license acceptance (%s)", matchedSource.Name, matchedSource.License)
			log.Printf("Use --force to accept license terms, or 'skyview-data import %s' for interactive acceptance", sourceName)
			continue
		}

		// Set license acceptance for forced operations
		if force && matchedSource.RequiresConsent {
			matchedSource.UserAcceptedLicense = true
			log.Printf("Accepting license terms for %s (%s)", matchedSource.Name, matchedSource.License)
		}

		log.Printf("Loading %s...", matchedSource.Name)
		
		result, err := loader.LoadDataSource(*matchedSource)
		if err != nil {
			log.Printf("Failed to load %s: %v", matchedSource.Name, err)
			continue
		}

		log.Printf("Loaded %s: %d records in %v", matchedSource.Name, result.RecordsNew, result.Duration)
		
		if len(result.Errors) > 0 {
			log.Printf("  %d errors occurred during import (first few):", len(result.Errors))
			for i, errMsg := range result.Errors {
				if i >= 3 { break }
				log.Printf("    %s", errMsg)
			}
		}
	}

	log.Println("Update completed successfully")
	return nil
}

// cmdImport imports data from a specific source with interactive license acceptance
func cmdImport(db *database.Database, sourceName string, force bool) error {
	availableSources := database.GetAvailableDataSources()
	
	var matchedSource *database.DataSource
	for _, available := range availableSources {
		if strings.EqualFold(strings.ReplaceAll(available.Name, " ", ""), sourceName) {
			matchedSource = &available
			break
		}
	}
	
	if matchedSource == nil {
		return fmt.Errorf("unknown data source: %s", sourceName)
	}

	// Handle license acceptance
	if matchedSource.RequiresConsent && !force {
		fmt.Printf("📄 License Information for %s\n", matchedSource.Name)
		fmt.Printf("   License: %s\n", matchedSource.License)
		fmt.Printf("   URL: %s\n", matchedSource.URL)
		fmt.Println()
		fmt.Printf("By importing this data, you agree to comply with the %s license terms.\n", matchedSource.License)
		
		if !askForConfirmation("Do you accept the license terms?") {
			fmt.Println("Import cancelled.")
			return nil
		}
		
		matchedSource.UserAcceptedLicense = true
	}

	fmt.Printf("📥 Importing %s...\n", matchedSource.Name)
	
	loader := database.NewDataLoader(db.GetConnection())
	result, err := loader.LoadDataSource(*matchedSource)
	if err != nil {
		return fmt.Errorf("import failed: %v", err)
	}

	fmt.Printf("✅ Import completed!\n")
	fmt.Printf("   Records: %d loaded, %d errors\n", result.RecordsNew, result.RecordsError)
	fmt.Printf("   Duration: %v\n", result.Duration)

	return nil
}

// cmdClear removes data from a specific source
func cmdClear(db *database.Database, sourceName string, force bool) error {
	if !force && !askForConfirmation(fmt.Sprintf("Clear all data from source '%s'?", sourceName)) {
		fmt.Println("Operation cancelled.")
		return nil
	}

	loader := database.NewDataLoader(db.GetConnection())
	err := loader.ClearDataSource(sourceName)
	if err != nil {
		return fmt.Errorf("clear failed: %v", err)
	}

	fmt.Printf("✅ Cleared data from source: %s\n", sourceName)
	return nil
}

// cmdReset clears all data and resets the database
func cmdReset(db *database.Database, force bool) error {
	if !force {
		fmt.Println("⚠️  This will remove ALL data from the database!")
		if !askForConfirmation("Are you sure you want to reset the database?") {
			fmt.Println("Reset cancelled.")
			return nil
		}
	}

	// This would require implementing a database reset function
	fmt.Println("❌ Database reset not yet implemented")
	return fmt.Errorf("reset functionality not implemented")
}

// askForConfirmation asks the user for yes/no confirmation
func askForConfirmation(question string) bool {
	fmt.Printf("%s (y/N): ", question)
	
	var response string
	fmt.Scanln(&response)
	
	response = strings.ToLower(strings.TrimSpace(response))
	return response == "y" || response == "yes"
}

// loadConfig loads the shared configuration file
func loadConfig(configPath string) (*Config, error) {
	data, err := os.ReadFile(configPath)
	if err != nil {
		return nil, fmt.Errorf("failed to read config file %s: %w", configPath, err)
	}
	
	var config Config
	if err := json.Unmarshal(data, &config); err != nil {
		return nil, fmt.Errorf("failed to parse config file %s: %w", configPath, err)
	}
	
	return &config, nil
}

// initDatabaseFromConfig initializes database using shared configuration
func initDatabaseFromConfig(config *Config, dbPathOverride string) (*database.Database, error) {
	var dbConfig *database.Config
	
	if config.Database != nil {
		dbConfig = config.Database
	} else {
		dbConfig = database.DefaultConfig()
	}
	
	// Allow command-line override of database path
	if dbPathOverride != "" {
		dbConfig.Path = dbPathOverride
	}
	
	// Resolve database path if empty
	if dbConfig.Path == "" {
		resolvedPath, err := database.ResolveDatabasePath(dbConfig.Path)
		if err != nil {
			return nil, fmt.Errorf("failed to resolve database path: %w", err)
		}
		dbConfig.Path = resolvedPath
	}
	
	// Create and initialize database
	db, err := database.NewDatabase(dbConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to create database: %w", err)
	}
	
	if err := db.Initialize(); err != nil {
		db.Close()
		return nil, fmt.Errorf("failed to initialize database: %w", err)
	}
	
	return db, nil
}

// cmdOptimize optimizes the database for storage efficiency
func cmdOptimize(db *database.Database, force bool) error {
	fmt.Println("Database Storage Optimization")
	fmt.Println("============================")
	
	// We need to get the database path from the config
	// For now, let's create a simple optimization manager
	config := &database.Config{
		Path: "./dev-skyview.db", // Default path - this should be configurable
	}
	
	// Create optimization manager
	optimizer := database.NewOptimizationManager(db, config)
	
	// Get current stats
	fmt.Println("📊 Current Database Statistics:")
	stats, err := optimizer.GetOptimizationStats()
	if err != nil {
		return fmt.Errorf("failed to get database stats: %w", err)
	}
	
	fmt.Printf("   • Size: %.1f MB\n", float64(stats.DatabaseSize)/(1024*1024))
	fmt.Printf("   • Page Size: %d bytes\n", stats.PageSize)
	fmt.Printf("   • Total Pages: %d\n", stats.PageCount)
	fmt.Printf("   • Used Pages: %d\n", stats.UsedPages)
	fmt.Printf("   • Free Pages: %d\n", stats.FreePages)
	fmt.Printf("   • Efficiency: %.1f%%\n", stats.Efficiency)
	fmt.Printf("   • Auto VACUUM: %v\n", stats.AutoVacuumEnabled)
	
	// Check if optimization is needed
	needsOptimization := stats.FreePages > 0 || stats.Efficiency < 95.0
	
	if !needsOptimization && !force {
		fmt.Println("✅ Database is already well optimized!")
		fmt.Println("   Use --force to run optimization anyway")
		return nil
	}
	
	// Perform optimizations
	if force && !needsOptimization {
		fmt.Println("\n🔧 Force optimization requested:")
	} else {
		fmt.Println("\n🔧 Applying Optimizations:")
	}
	
	if err := optimizer.VacuumDatabase(); err != nil {
		return fmt.Errorf("VACUUM failed: %w", err)
	}
	
	if err := optimizer.OptimizeDatabase(); err != nil {
		return fmt.Errorf("optimization failed: %w", err)
	}
	
	// Show final stats
	fmt.Println("\n📈 Final Statistics:")
	finalStats, err := optimizer.GetOptimizationStats()
	if err != nil {
		return fmt.Errorf("failed to get final stats: %w", err)
	}
	
	fmt.Printf("   • Size: %.1f MB\n", float64(finalStats.DatabaseSize)/(1024*1024))
	fmt.Printf("   • Efficiency: %.1f%%\n", finalStats.Efficiency)
	fmt.Printf("   • Free Pages: %d\n", finalStats.FreePages)
	
	if stats.DatabaseSize > finalStats.DatabaseSize {
		saved := stats.DatabaseSize - finalStats.DatabaseSize
		fmt.Printf("   • Space Saved: %.1f MB\n", float64(saved)/(1024*1024))
	}
	
	fmt.Println("\n✅ Database optimization completed!")
	return nil
}