// Package main implements the SkyView data management utility. // // This tool provides simple commands for populating and updating the SkyView // database with aviation data from various external sources while maintaining // proper license compliance. package main import ( "encoding/json" "flag" "fmt" "log" "os" "strings" "time" "skyview/internal/database" ) // Shared configuration structures (should match main skyview) type Config struct { Server ServerConfig `json:"server"` Sources []SourceConfig `json:"sources"` Settings Settings `json:"settings"` Database *database.Config `json:"database,omitempty"` Callsign *CallsignConfig `json:"callsign,omitempty"` Origin OriginConfig `json:"origin"` } type CallsignConfig struct { Enabled bool `json:"enabled"` CacheHours int `json:"cache_hours"` PrivacyMode bool `json:"privacy_mode"` Sources map[string]CallsignSourceConfig `json:"sources"` ExternalAPIs map[string]ExternalAPIConfig `json:"external_apis,omitempty"` } type CallsignSourceConfig struct { Enabled bool `json:"enabled"` Priority int `json:"priority"` License string `json:"license"` RequiresConsent bool `json:"requires_consent,omitempty"` UserAcceptsTerms bool `json:"user_accepts_terms,omitempty"` } type ExternalAPIConfig struct { Enabled bool `json:"enabled"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` MaxRetries int `json:"max_retries,omitempty"` RequiresConsent bool `json:"requires_consent,omitempty"` } type OriginConfig struct { Latitude float64 `json:"latitude"` Longitude float64 `json:"longitude"` Name string `json:"name,omitempty"` } type ServerConfig struct { Host string `json:"host"` Port int `json:"port"` } type SourceConfig struct { ID string `json:"id"` Name string `json:"name"` Host string `json:"host"` Port int `json:"port"` Format string `json:"format,omitempty"` Latitude float64 `json:"latitude"` Longitude float64 `json:"longitude"` Altitude float64 `json:"altitude"` Enabled bool `json:"enabled"` } type Settings struct { HistoryLimit int `json:"history_limit"` StaleTimeout int `json:"stale_timeout"` UpdateRate int `json:"update_rate"` } // Version information var ( version = "dev" commit = "unknown" date = "unknown" ) func main() { var ( configPath = flag.String("config", "config.json", "Configuration file path") dbPath = flag.String("db", "", "Database file path (override config)") verbose = flag.Bool("v", false, "Verbose output") force = flag.Bool("force", false, "Force operation without prompts") showVer = flag.Bool("version", false, "Show version information") ) flag.Usage = func() { fmt.Fprintf(os.Stderr, `SkyView Data Manager v%s USAGE: skyview-data [OPTIONS] COMMAND [ARGS...] COMMANDS: init Initialize empty database list List available data sources status Show current database status update [SOURCE...] Update data from sources (default: safe sources) import SOURCE Import data from specific source clear SOURCE Remove data from specific source reset Clear all data and reset database optimize Optimize database for storage efficiency EXAMPLES: skyview-data init # Create empty database skyview-data update # Update from safe (public domain) sources skyview-data update openflights # Update OpenFlights data (requires license acceptance) skyview-data import ourairports # Import OurAirports data skyview-data list # Show available sources skyview-data status # Show database status skyview-data optimize # Optimize database storage OPTIONS: `, version) flag.PrintDefaults() } flag.Parse() if *showVer { fmt.Printf("skyview-data version %s (commit %s, built %s)\n", version, commit, date) return } if flag.NArg() == 0 { flag.Usage() os.Exit(1) } command := flag.Arg(0) // Set up logging for cron-friendly operation if *verbose { log.SetFlags(log.LstdFlags | log.Lshortfile) } else { // For cron jobs, include timestamp but no file info log.SetFlags(log.LstdFlags) } // Load configuration config, err := loadConfig(*configPath) if err != nil { log.Fatalf("Configuration loading failed: %v", err) } // Initialize database connection using shared config db, err := initDatabaseFromConfig(config, *dbPath) if err != nil { log.Fatalf("Database initialization failed: %v", err) } defer db.Close() // Execute command switch command { case "init": err = cmdInit(db, *force) case "list": err = cmdList(db) case "status": err = cmdStatus(db) case "update": sources := flag.Args()[1:] err = cmdUpdate(db, sources, *force) case "import": if flag.NArg() < 2 { log.Fatal("import command requires a source name") } err = cmdImport(db, flag.Arg(1), *force) case "clear": if flag.NArg() < 2 { log.Fatal("clear command requires a source name") } err = cmdClear(db, flag.Arg(1), *force) case "reset": err = cmdReset(db, *force) case "optimize": err = cmdOptimize(db, *force) default: log.Fatalf("Unknown command: %s", command) } if err != nil { log.Fatalf("Command failed: %v", err) } } // initDatabase initializes the database connection with auto-creation func initDatabase(dbPath string) (*database.Database, error) { config := database.DefaultConfig() config.Path = dbPath db, err := database.NewDatabase(config) if err != nil { return nil, fmt.Errorf("failed to create database: %v", err) } if err := db.Initialize(); err != nil { db.Close() return nil, fmt.Errorf("failed to initialize database: %v", err) } return db, nil } // cmdInit initializes an empty database func cmdInit(db *database.Database, force bool) error { dbPath := db.GetConfig().Path // Check if database already exists and has data if !force { if stats, err := db.GetHistoryManager().GetStatistics(); err == nil { if totalRecords, ok := stats["total_records"].(int); ok && totalRecords > 0 { fmt.Printf("Database already exists with %d records at: %s\n", totalRecords, dbPath) fmt.Println("Use --force to reinitialize") return nil } } } fmt.Printf("Initializing SkyView database at: %s\n", dbPath) fmt.Println("āœ“ Database schema created") fmt.Println("āœ“ Empty tables ready for data import") fmt.Println() fmt.Println("Next steps:") fmt.Println(" skyview-data update # Import safe (public domain) data") fmt.Println(" skyview-data list # Show available data sources") fmt.Println(" skyview-data status # Check database status") return nil } // cmdList shows available data sources func cmdList(db *database.Database) error { fmt.Println("Available Data Sources:") fmt.Println() sources := database.GetAvailableDataSources() for _, source := range sources { status := "🟢" if source.RequiresConsent { status = "āš ļø " } fmt.Printf("%s %s\n", status, source.Name) fmt.Printf(" License: %s\n", source.License) fmt.Printf(" URL: %s\n", source.URL) if source.RequiresConsent { fmt.Printf(" Note: Requires license acceptance\n") } fmt.Println() } fmt.Println("Legend:") fmt.Println(" 🟢 = Safe to use automatically (Public Domain/MIT)") fmt.Println(" āš ļø = Requires license acceptance (AGPL, etc.)") return nil } // cmdStatus shows current database status func cmdStatus(db *database.Database) error { fmt.Println("SkyView Database Status") fmt.Println("======================") dbPath := db.GetConfig().Path fmt.Printf("Database: %s\n", dbPath) // Check if file exists and get size if stat, err := os.Stat(dbPath); err == nil { fmt.Printf("Size: %.2f MB\n", float64(stat.Size())/(1024*1024)) fmt.Printf("Modified: %s\n", stat.ModTime().Format(time.RFC3339)) // Add database optimization stats optimizer := database.NewOptimizationManager(db, db.GetConfig()) if stats, err := optimizer.GetOptimizationStats(); err == nil { fmt.Printf("Efficiency: %.1f%% (%d used pages, %d free pages)\n", stats.Efficiency, stats.UsedPages, stats.FreePages) if stats.AutoVacuumEnabled { fmt.Printf("Auto-VACUUM: Enabled\n") } } } fmt.Println() // Show loaded data sources loader := database.NewDataLoader(db.GetConnection()) loadedSources, err := loader.GetLoadedDataSources() if err != nil { return fmt.Errorf("failed to get loaded sources: %v", err) } if len(loadedSources) == 0 { fmt.Println("šŸ“­ No data sources loaded") fmt.Println(" Run 'skyview-data update' to populate with aviation data") } else { fmt.Printf("šŸ“¦ Loaded Data Sources (%d):\n", len(loadedSources)) for _, source := range loadedSources { fmt.Printf(" • %s (%s)\n", source.Name, source.License) } } fmt.Println() // Show statistics stats, err := db.GetHistoryManager().GetStatistics() if err != nil { return fmt.Errorf("failed to get statistics: %v", err) } // Show comprehensive reference data statistics var airportCount, airlineCount int db.GetConnection().QueryRow(`SELECT COUNT(*) FROM airports`).Scan(&airportCount) db.GetConnection().QueryRow(`SELECT COUNT(*) FROM airlines`).Scan(&airlineCount) // Get data source update information var lastUpdate time.Time var updateCount int err = db.GetConnection().QueryRow(` SELECT COUNT(*), MAX(imported_at) FROM data_sources WHERE imported_at IS NOT NULL `).Scan(&updateCount, &lastUpdate) fmt.Printf("šŸ“Š Database Statistics:\n") fmt.Printf(" Reference Data:\n") if airportCount > 0 { fmt.Printf(" • Airports: %d\n", airportCount) } if airlineCount > 0 { fmt.Printf(" • Airlines: %d\n", airlineCount) } if updateCount > 0 { fmt.Printf(" • Data Sources: %d imported\n", updateCount) if !lastUpdate.IsZero() { fmt.Printf(" • Last Updated: %s\n", lastUpdate.Format("2006-01-02 15:04:05")) } } fmt.Printf(" Flight History:\n") if totalRecords, ok := stats["total_records"].(int); ok { fmt.Printf(" • Aircraft Records: %d\n", totalRecords) } if uniqueAircraft, ok := stats["unique_aircraft"].(int); ok { fmt.Printf(" • Unique Aircraft: %d\n", uniqueAircraft) } if recentRecords, ok := stats["recent_records_24h"].(int); ok { fmt.Printf(" • Last 24h: %d records\n", recentRecords) } oldestRecord, hasOldest := stats["oldest_record"] newestRecord, hasNewest := stats["newest_record"] if hasOldest && hasNewest && oldestRecord != nil && newestRecord != nil { if oldest, ok := oldestRecord.(time.Time); ok { if newest, ok := newestRecord.(time.Time); ok { fmt.Printf(" • Flight Data Range: %s to %s\n", oldest.Format("2006-01-02"), newest.Format("2006-01-02")) } } } // Show airport data sample if available if airportCount > 0 { var sampleAirports []string rows, err := db.GetConnection().Query(` SELECT name || ' (' || COALESCE(icao_code, ident) || ')' FROM airports WHERE icao_code IS NOT NULL AND icao_code != '' ORDER BY name LIMIT 3 `) if err == nil { for rows.Next() { var airport string if rows.Scan(&airport) == nil { sampleAirports = append(sampleAirports, airport) } } rows.Close() if len(sampleAirports) > 0 { fmt.Printf(" • Sample Airports: %s\n", strings.Join(sampleAirports, ", ")) } } } return nil } // cmdUpdate updates data from specified sources (or safe sources by default) func cmdUpdate(db *database.Database, sources []string, force bool) error { availableSources := database.GetAvailableDataSources() // If no sources specified, use safe (non-consent-required) sources if len(sources) == 0 { log.Println("Updating from safe data sources...") for _, source := range availableSources { if !source.RequiresConsent { sources = append(sources, strings.ToLower(strings.ReplaceAll(source.Name, " ", ""))) } } if len(sources) == 0 { log.Println("No safe data sources available for automatic update") return nil } log.Printf("Found %d safe data sources to update", len(sources)) } loader := database.NewDataLoader(db.GetConnection()) for _, sourceName := range sources { // Find matching source var matchedSource *database.DataSource for _, available := range availableSources { if strings.EqualFold(strings.ReplaceAll(available.Name, " ", ""), sourceName) { matchedSource = &available break } } if matchedSource == nil { log.Printf("āš ļø Unknown source: %s", sourceName) continue } // Check for consent requirement if matchedSource.RequiresConsent && !force { log.Printf("Skipping %s: requires license acceptance (%s)", matchedSource.Name, matchedSource.License) log.Printf("Use --force to accept license terms, or 'skyview-data import %s' for interactive acceptance", sourceName) continue } // Set license acceptance for forced operations if force && matchedSource.RequiresConsent { matchedSource.UserAcceptedLicense = true log.Printf("Accepting license terms for %s (%s)", matchedSource.Name, matchedSource.License) } log.Printf("Loading %s...", matchedSource.Name) result, err := loader.LoadDataSource(*matchedSource) if err != nil { log.Printf("Failed to load %s: %v", matchedSource.Name, err) continue } log.Printf("Loaded %s: %d records in %v", matchedSource.Name, result.RecordsNew, result.Duration) if len(result.Errors) > 0 { log.Printf(" %d errors occurred during import (first few):", len(result.Errors)) for i, errMsg := range result.Errors { if i >= 3 { break } log.Printf(" %s", errMsg) } } } log.Println("Update completed successfully") return nil } // cmdImport imports data from a specific source with interactive license acceptance func cmdImport(db *database.Database, sourceName string, force bool) error { availableSources := database.GetAvailableDataSources() var matchedSource *database.DataSource for _, available := range availableSources { if strings.EqualFold(strings.ReplaceAll(available.Name, " ", ""), sourceName) { matchedSource = &available break } } if matchedSource == nil { return fmt.Errorf("unknown data source: %s", sourceName) } // Handle license acceptance if matchedSource.RequiresConsent && !force { fmt.Printf("šŸ“„ License Information for %s\n", matchedSource.Name) fmt.Printf(" License: %s\n", matchedSource.License) fmt.Printf(" URL: %s\n", matchedSource.URL) fmt.Println() fmt.Printf("By importing this data, you agree to comply with the %s license terms.\n", matchedSource.License) if !askForConfirmation("Do you accept the license terms?") { fmt.Println("Import cancelled.") return nil } matchedSource.UserAcceptedLicense = true } fmt.Printf("šŸ“„ Importing %s...\n", matchedSource.Name) loader := database.NewDataLoader(db.GetConnection()) result, err := loader.LoadDataSource(*matchedSource) if err != nil { return fmt.Errorf("import failed: %v", err) } fmt.Printf("āœ… Import completed!\n") fmt.Printf(" Records: %d loaded, %d errors\n", result.RecordsNew, result.RecordsError) fmt.Printf(" Duration: %v\n", result.Duration) return nil } // cmdClear removes data from a specific source func cmdClear(db *database.Database, sourceName string, force bool) error { if !force && !askForConfirmation(fmt.Sprintf("Clear all data from source '%s'?", sourceName)) { fmt.Println("Operation cancelled.") return nil } loader := database.NewDataLoader(db.GetConnection()) err := loader.ClearDataSource(sourceName) if err != nil { return fmt.Errorf("clear failed: %v", err) } fmt.Printf("āœ… Cleared data from source: %s\n", sourceName) return nil } // cmdReset clears all data and resets the database func cmdReset(db *database.Database, force bool) error { if !force { fmt.Println("āš ļø This will remove ALL data from the database!") if !askForConfirmation("Are you sure you want to reset the database?") { fmt.Println("Reset cancelled.") return nil } } // This would require implementing a database reset function fmt.Println("āŒ Database reset not yet implemented") return fmt.Errorf("reset functionality not implemented") } // askForConfirmation asks the user for yes/no confirmation func askForConfirmation(question string) bool { fmt.Printf("%s (y/N): ", question) var response string fmt.Scanln(&response) response = strings.ToLower(strings.TrimSpace(response)) return response == "y" || response == "yes" } // loadConfig loads the shared configuration file func loadConfig(configPath string) (*Config, error) { data, err := os.ReadFile(configPath) if err != nil { return nil, fmt.Errorf("failed to read config file %s: %w", configPath, err) } var config Config if err := json.Unmarshal(data, &config); err != nil { return nil, fmt.Errorf("failed to parse config file %s: %w", configPath, err) } return &config, nil } // initDatabaseFromConfig initializes database using shared configuration func initDatabaseFromConfig(config *Config, dbPathOverride string) (*database.Database, error) { var dbConfig *database.Config if config.Database != nil { dbConfig = config.Database } else { dbConfig = database.DefaultConfig() } // Allow command-line override of database path if dbPathOverride != "" { dbConfig.Path = dbPathOverride } // Resolve database path if empty if dbConfig.Path == "" { resolvedPath, err := database.ResolveDatabasePath(dbConfig.Path) if err != nil { return nil, fmt.Errorf("failed to resolve database path: %w", err) } dbConfig.Path = resolvedPath } // Create and initialize database db, err := database.NewDatabase(dbConfig) if err != nil { return nil, fmt.Errorf("failed to create database: %w", err) } if err := db.Initialize(); err != nil { db.Close() return nil, fmt.Errorf("failed to initialize database: %w", err) } return db, nil } // cmdOptimize optimizes the database for storage efficiency func cmdOptimize(db *database.Database, force bool) error { fmt.Println("Database Storage Optimization") fmt.Println("============================") // We need to get the database path from the config // For now, let's create a simple optimization manager config := &database.Config{ Path: "./dev-skyview.db", // Default path - this should be configurable } // Create optimization manager optimizer := database.NewOptimizationManager(db, config) // Get current stats fmt.Println("šŸ“Š Current Database Statistics:") stats, err := optimizer.GetOptimizationStats() if err != nil { return fmt.Errorf("failed to get database stats: %w", err) } fmt.Printf(" • Size: %.1f MB\n", float64(stats.DatabaseSize)/(1024*1024)) fmt.Printf(" • Page Size: %d bytes\n", stats.PageSize) fmt.Printf(" • Total Pages: %d\n", stats.PageCount) fmt.Printf(" • Used Pages: %d\n", stats.UsedPages) fmt.Printf(" • Free Pages: %d\n", stats.FreePages) fmt.Printf(" • Efficiency: %.1f%%\n", stats.Efficiency) fmt.Printf(" • Auto VACUUM: %v\n", stats.AutoVacuumEnabled) // Check if optimization is needed needsOptimization := stats.FreePages > 0 || stats.Efficiency < 95.0 if !needsOptimization && !force { fmt.Println("āœ… Database is already well optimized!") fmt.Println(" Use --force to run optimization anyway") return nil } // Perform optimizations if force && !needsOptimization { fmt.Println("\nšŸ”§ Force optimization requested:") } else { fmt.Println("\nšŸ”§ Applying Optimizations:") } if err := optimizer.VacuumDatabase(); err != nil { return fmt.Errorf("VACUUM failed: %w", err) } if err := optimizer.OptimizeDatabase(); err != nil { return fmt.Errorf("optimization failed: %w", err) } // Show final stats fmt.Println("\nšŸ“ˆ Final Statistics:") finalStats, err := optimizer.GetOptimizationStats() if err != nil { return fmt.Errorf("failed to get final stats: %w", err) } fmt.Printf(" • Size: %.1f MB\n", float64(finalStats.DatabaseSize)/(1024*1024)) fmt.Printf(" • Efficiency: %.1f%%\n", finalStats.Efficiency) fmt.Printf(" • Free Pages: %d\n", finalStats.FreePages) if stats.DatabaseSize > finalStats.DatabaseSize { saved := stats.DatabaseSize - finalStats.DatabaseSize fmt.Printf(" • Space Saved: %.1f MB\n", float64(saved)/(1024*1024)) } fmt.Println("\nāœ… Database optimization completed!") return nil }