diff --git a/.gitignore b/.gitignore index aedbbcb..20e7e00 100644 --- a/.gitignore +++ b/.gitignore @@ -36,12 +36,4 @@ Thumbs.db # Temporary files tmp/ -temp/ - -# Database files -*.db -*.db-shm -*.db-wal -*.sqlite -*.sqlite3 -dev-skyview.db \ No newline at end of file +temp/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index c8b46c9..c7a8ea6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,5 +103,4 @@ This document outlines coding standards, architectural principles, and developme --- -These guidelines ensure SkyView remains reliable, maintainable, and suitable for aviation use while supporting continued development and enhancement. -- All future changes to the UX should keep in mind a gradual move to a mobile friendly design, but not at the cost of a working and useful UI for non-mobile clients. \ No newline at end of file +These guidelines ensure SkyView remains reliable, maintainable, and suitable for aviation use while supporting continued development and enhancement. \ No newline at end of file diff --git a/Makefile b/Makefile index 1543a96..2f71042 100644 --- a/Makefile +++ b/Makefile @@ -17,14 +17,8 @@ build-beast-dump: @mkdir -p $(BUILD_DIR) go build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/beast-dump ./cmd/beast-dump -# Build skyview-data database management binary -build-skyview-data: - @echo "Building skyview-data..." - @mkdir -p $(BUILD_DIR) - go build -ldflags="$(LDFLAGS)" -o $(BUILD_DIR)/skyview-data ./cmd/skyview-data - # Build all binaries -build-all: build build-beast-dump build-skyview-data +build-all: build build-beast-dump @echo "Built all binaries successfully:" @ls -la $(BUILD_DIR)/ @@ -105,4 +99,4 @@ vet: check: format vet lint test @echo "All checks passed!" -.DEFAULT_GOAL := build-all \ No newline at end of file +.DEFAULT_GOAL := build \ No newline at end of file diff --git a/README.md b/README.md index a034158..3765f15 100644 --- a/README.md +++ b/README.md @@ -27,15 +27,12 @@ A high-performance, multi-source ADS-B aircraft tracking application that connec - **Map Controls**: Center on aircraft, reset to origin, toggle overlays - **Signal Heatmaps**: Coverage heatmap visualization *(under construction)* šŸš§ -### Aircraft Data Enhancement +### Aircraft Data - **Complete Mode S Decoding**: Position, velocity, altitude, heading - **Aircraft Identification**: Callsign, category, country, registration -- **Enhanced Callsign Lookup**: Multi-source airline database with 6,162+ airlines and 83,557+ airports -- **Aviation Data Integration**: OpenFlights and OurAirports databases with automatic updates - **ICAO Country Database**: Comprehensive embedded database with 70+ allocations covering 40+ countries - **Multi-source Tracking**: Signal strength from each receiver -- **Historical Data**: Position history with configurable retention -- **Database Optimization**: Automatic VACUUM operations and storage efficiency monitoring +- **Historical Data**: Position history and trail visualization ## šŸš€ Quick Start @@ -258,66 +255,23 @@ sudo journalctl -u skyview -f make build # Create user and directories -sudo useradd -r -s /bin/false skyview-adsb -sudo mkdir -p /etc/skyview-adsb /var/lib/skyview-adsb /var/log/skyview-adsb -sudo chown skyview-adsb:skyview-adsb /var/lib/skyview-adsb /var/log/skyview-adsb +sudo useradd -r -s /bin/false skyview +sudo mkdir -p /etc/skyview /var/lib/skyview /var/log/skyview +sudo chown skyview:skyview /var/lib/skyview /var/log/skyview # Install binary and config sudo cp build/skyview /usr/bin/ -sudo cp build/skyview-data /usr/bin/ -sudo cp config.example.json /etc/skyview-adsb/config.json -sudo chown root:skyview-adsb /etc/skyview-adsb/config.json -sudo chmod 640 /etc/skyview-adsb/config.json +sudo cp config.example.json /etc/skyview/config.json +sudo chown root:skyview /etc/skyview/config.json +sudo chmod 640 /etc/skyview/config.json # Create systemd service -sudo cp debian/lib/systemd/system/skyview-adsb.service /lib/systemd/system/ -sudo cp debian/lib/systemd/system/skyview-database-update.service /lib/systemd/system/ -sudo cp debian/lib/systemd/system/skyview-database-update.timer /lib/systemd/system/ +sudo cp debian/lib/systemd/system/skyview.service /lib/systemd/system/ sudo systemctl daemon-reload -sudo systemctl enable skyview-adsb -sudo systemctl enable skyview-database-update.timer -sudo systemctl start skyview-adsb -sudo systemctl start skyview-database-update.timer +sudo systemctl enable skyview +sudo systemctl start skyview ``` -### Database Management - -SkyView includes powerful database management capabilities through the `skyview-data` command: - -```bash -# Update aviation data sources (airlines, airports) -skyview-data update - -# Optimize database storage and performance -skyview-data optimize - -# Check database optimization statistics -skyview-data optimize --stats-only - -# List available data sources -skyview-data list - -# Check current database status -skyview-data status -``` - -The system automatically: -- Updates aviation databases on service startup -- Runs weekly database updates via systemd timer -- Optimizes storage with VACUUM operations -- Monitors database efficiency and statistics - -### Configuration - -SkyView supports comprehensive configuration including external aviation data sources: - -- **3 External Data Sources**: OpenFlights Airlines (~6,162), OpenFlights Airports (~7,698), OurAirports (~83,557) -- **Database Management**: Automatic optimization, configurable retention, backup settings -- **Privacy Controls**: Privacy mode for air-gapped operation, selective source control -- **Performance Tuning**: Connection pooling, cache settings, update intervals - -See **[Configuration Guide](docs/CONFIGURATION.md)** for complete documentation of all options. - ## šŸ”’ Security The application includes security hardening: diff --git a/assets/static/database.html b/assets/static/database.html deleted file mode 100644 index cec9db8..0000000 --- a/assets/static/database.html +++ /dev/null @@ -1,360 +0,0 @@ - - - - - - Database Status - SkyView - - - - - -
-

SkyView Database Status

-
- ā† Back to Map - šŸ”„ Refresh -
-
-
- -
- -
-

šŸ“Š Database Statistics

-
Loading database statistics...
-
- - -
-

šŸ“¦ External Data Sources

-
Loading data sources...
-
-
- - - - \ No newline at end of file diff --git a/cmd/skyview-data/main.go b/cmd/skyview-data/main.go deleted file mode 100644 index e7bb5f9..0000000 --- a/cmd/skyview-data/main.go +++ /dev/null @@ -1,686 +0,0 @@ -// Package main implements the SkyView data management utility. -// -// This tool provides simple commands for populating and updating the SkyView -// database with aviation data from various external sources while maintaining -// proper license compliance. -package main - -import ( - "encoding/json" - "flag" - "fmt" - "log" - "os" - "strings" - "time" - - "skyview/internal/database" -) - -// Shared configuration structures (should match main skyview) -type Config struct { - Server ServerConfig `json:"server"` - Sources []SourceConfig `json:"sources"` - Settings Settings `json:"settings"` - Database *database.Config `json:"database,omitempty"` - Callsign *CallsignConfig `json:"callsign,omitempty"` - Origin OriginConfig `json:"origin"` -} - -type CallsignConfig struct { - Enabled bool `json:"enabled"` - CacheHours int `json:"cache_hours"` - PrivacyMode bool `json:"privacy_mode"` - Sources map[string]CallsignSourceConfig `json:"sources"` - ExternalAPIs map[string]ExternalAPIConfig `json:"external_apis,omitempty"` -} - -type CallsignSourceConfig struct { - Enabled bool `json:"enabled"` - Priority int `json:"priority"` - License string `json:"license"` - RequiresConsent bool `json:"requires_consent,omitempty"` - UserAcceptsTerms bool `json:"user_accepts_terms,omitempty"` -} - -type ExternalAPIConfig struct { - Enabled bool `json:"enabled"` - TimeoutSeconds int `json:"timeout_seconds,omitempty"` - MaxRetries int `json:"max_retries,omitempty"` - RequiresConsent bool `json:"requires_consent,omitempty"` -} - -type OriginConfig struct { - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Name string `json:"name,omitempty"` -} - -type ServerConfig struct { - Host string `json:"host"` - Port int `json:"port"` -} - -type SourceConfig struct { - ID string `json:"id"` - Name string `json:"name"` - Host string `json:"host"` - Port int `json:"port"` - Format string `json:"format,omitempty"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Altitude float64 `json:"altitude"` - Enabled bool `json:"enabled"` -} - -type Settings struct { - HistoryLimit int `json:"history_limit"` - StaleTimeout int `json:"stale_timeout"` - UpdateRate int `json:"update_rate"` -} - -// Version information -var ( - version = "dev" - commit = "unknown" - date = "unknown" -) - -func main() { - var ( - configPath = flag.String("config", "config.json", "Configuration file path") - dbPath = flag.String("db", "", "Database file path (override config)") - verbose = flag.Bool("v", false, "Verbose output") - force = flag.Bool("force", false, "Force operation without prompts") - showVer = flag.Bool("version", false, "Show version information") - ) - - flag.Usage = func() { - fmt.Fprintf(os.Stderr, `SkyView Data Manager v%s - -USAGE: - skyview-data [OPTIONS] COMMAND [ARGS...] - -COMMANDS: - init Initialize empty database - list List available data sources - status Show current database status - update [SOURCE...] Update data from sources (default: safe sources) - import SOURCE Import data from specific source - clear SOURCE Remove data from specific source - reset Clear all data and reset database - optimize Optimize database for storage efficiency - -EXAMPLES: - skyview-data init # Create empty database - skyview-data update # Update from safe (public domain) sources - skyview-data update openflights # Update OpenFlights data (requires license acceptance) - skyview-data import ourairports # Import OurAirports data - skyview-data list # Show available sources - skyview-data status # Show database status - skyview-data optimize # Optimize database storage - -OPTIONS: -`, version) - flag.PrintDefaults() - } - - flag.Parse() - - if *showVer { - fmt.Printf("skyview-data version %s (commit %s, built %s)\n", version, commit, date) - return - } - - if flag.NArg() == 0 { - flag.Usage() - os.Exit(1) - } - - command := flag.Arg(0) - - // Set up logging for cron-friendly operation - if *verbose { - log.SetFlags(log.LstdFlags | log.Lshortfile) - } else { - // For cron jobs, include timestamp but no file info - log.SetFlags(log.LstdFlags) - } - - // Load configuration - config, err := loadConfig(*configPath) - if err != nil { - log.Fatalf("Configuration loading failed: %v", err) - } - - // Initialize database connection using shared config - db, err := initDatabaseFromConfig(config, *dbPath) - if err != nil { - log.Fatalf("Database initialization failed: %v", err) - } - defer db.Close() - - // Execute command - switch command { - case "init": - err = cmdInit(db, *force) - case "list": - err = cmdList(db) - case "status": - err = cmdStatus(db) - case "update": - sources := flag.Args()[1:] - err = cmdUpdate(db, sources, *force) - case "import": - if flag.NArg() < 2 { - log.Fatal("import command requires a source name") - } - err = cmdImport(db, flag.Arg(1), *force) - case "clear": - if flag.NArg() < 2 { - log.Fatal("clear command requires a source name") - } - err = cmdClear(db, flag.Arg(1), *force) - case "reset": - err = cmdReset(db, *force) - case "optimize": - err = cmdOptimize(db, *force) - default: - log.Fatalf("Unknown command: %s", command) - } - - if err != nil { - log.Fatalf("Command failed: %v", err) - } -} - -// initDatabase initializes the database connection with auto-creation -func initDatabase(dbPath string) (*database.Database, error) { - config := database.DefaultConfig() - config.Path = dbPath - - db, err := database.NewDatabase(config) - if err != nil { - return nil, fmt.Errorf("failed to create database: %v", err) - } - - if err := db.Initialize(); err != nil { - db.Close() - return nil, fmt.Errorf("failed to initialize database: %v", err) - } - - return db, nil -} - -// cmdInit initializes an empty database -func cmdInit(db *database.Database, force bool) error { - dbPath := db.GetConfig().Path - - // Check if database already exists and has data - if !force { - if stats, err := db.GetHistoryManager().GetStatistics(); err == nil { - if totalRecords, ok := stats["total_records"].(int); ok && totalRecords > 0 { - fmt.Printf("Database already exists with %d records at: %s\n", totalRecords, dbPath) - fmt.Println("Use --force to reinitialize") - return nil - } - } - } - - fmt.Printf("Initializing SkyView database at: %s\n", dbPath) - fmt.Println("āœ“ Database schema created") - fmt.Println("āœ“ Empty tables ready for data import") - fmt.Println() - fmt.Println("Next steps:") - fmt.Println(" skyview-data update # Import safe (public domain) data") - fmt.Println(" skyview-data list # Show available data sources") - fmt.Println(" skyview-data status # Check database status") - - return nil -} - -// cmdList shows available data sources -func cmdList(db *database.Database) error { - fmt.Println("Available Data Sources:") - fmt.Println() - - sources := database.GetAvailableDataSources() - for _, source := range sources { - status := "šŸŸ¢" - if source.RequiresConsent { - status = "āš ļø " - } - - fmt.Printf("%s %s\n", status, source.Name) - fmt.Printf(" License: %s\n", source.License) - fmt.Printf(" URL: %s\n", source.URL) - if source.RequiresConsent { - fmt.Printf(" Note: Requires license acceptance\n") - } - fmt.Println() - } - - fmt.Println("Legend:") - fmt.Println(" šŸŸ¢ = Safe to use automatically (Public Domain/MIT)") - fmt.Println(" āš ļø = Requires license acceptance (AGPL, etc.)") - - return nil -} - -// cmdStatus shows current database status -func cmdStatus(db *database.Database) error { - fmt.Println("SkyView Database Status") - fmt.Println("======================") - - dbPath := db.GetConfig().Path - fmt.Printf("Database: %s\n", dbPath) - - // Check if file exists and get size - if stat, err := os.Stat(dbPath); err == nil { - fmt.Printf("Size: %.2f MB\n", float64(stat.Size())/(1024*1024)) - fmt.Printf("Modified: %s\n", stat.ModTime().Format(time.RFC3339)) - - // Add database optimization stats - optimizer := database.NewOptimizationManager(db, db.GetConfig()) - if stats, err := optimizer.GetOptimizationStats(); err == nil { - fmt.Printf("Efficiency: %.1f%% (%d used pages, %d free pages)\n", - stats.Efficiency, stats.UsedPages, stats.FreePages) - if stats.AutoVacuumEnabled { - fmt.Printf("Auto-VACUUM: Enabled\n") - } - } - } - fmt.Println() - - // Show loaded data sources - loader := database.NewDataLoader(db.GetConnection()) - loadedSources, err := loader.GetLoadedDataSources() - if err != nil { - return fmt.Errorf("failed to get loaded sources: %v", err) - } - - if len(loadedSources) == 0 { - fmt.Println("šŸ“­ No data sources loaded") - fmt.Println(" Run 'skyview-data update' to populate with aviation data") - } else { - fmt.Printf("šŸ“¦ Loaded Data Sources (%d):\n", len(loadedSources)) - for _, source := range loadedSources { - fmt.Printf(" ā€¢ %s (%s)\n", source.Name, source.License) - } - } - fmt.Println() - - // Show statistics - stats, err := db.GetHistoryManager().GetStatistics() - if err != nil { - return fmt.Errorf("failed to get statistics: %v", err) - } - - // Show comprehensive reference data statistics - var airportCount, airlineCount int - db.GetConnection().QueryRow(`SELECT COUNT(*) FROM airports`).Scan(&airportCount) - db.GetConnection().QueryRow(`SELECT COUNT(*) FROM airlines`).Scan(&airlineCount) - - // Get data source update information - var lastUpdate time.Time - var updateCount int - err = db.GetConnection().QueryRow(` - SELECT COUNT(*), MAX(imported_at) - FROM data_sources - WHERE imported_at IS NOT NULL - `).Scan(&updateCount, &lastUpdate) - - fmt.Printf("šŸ“Š Database Statistics:\n") - fmt.Printf(" Reference Data:\n") - if airportCount > 0 { - fmt.Printf(" ā€¢ Airports: %d\n", airportCount) - } - if airlineCount > 0 { - fmt.Printf(" ā€¢ Airlines: %d\n", airlineCount) - } - if updateCount > 0 { - fmt.Printf(" ā€¢ Data Sources: %d imported\n", updateCount) - if !lastUpdate.IsZero() { - fmt.Printf(" ā€¢ Last Updated: %s\n", lastUpdate.Format("2006-01-02 15:04:05")) - } - } - - fmt.Printf(" Flight History:\n") - if totalRecords, ok := stats["total_records"].(int); ok { - fmt.Printf(" ā€¢ Aircraft Records: %d\n", totalRecords) - } - if uniqueAircraft, ok := stats["unique_aircraft"].(int); ok { - fmt.Printf(" ā€¢ Unique Aircraft: %d\n", uniqueAircraft) - } - if recentRecords, ok := stats["recent_records_24h"].(int); ok { - fmt.Printf(" ā€¢ Last 24h: %d records\n", recentRecords) - } - - oldestRecord, hasOldest := stats["oldest_record"] - newestRecord, hasNewest := stats["newest_record"] - if hasOldest && hasNewest && oldestRecord != nil && newestRecord != nil { - if oldest, ok := oldestRecord.(time.Time); ok { - if newest, ok := newestRecord.(time.Time); ok { - fmt.Printf(" ā€¢ Flight Data Range: %s to %s\n", - oldest.Format("2006-01-02"), - newest.Format("2006-01-02")) - } - } - } - - // Show airport data sample if available - if airportCount > 0 { - var sampleAirports []string - rows, err := db.GetConnection().Query(` - SELECT name || ' (' || COALESCE(icao_code, ident) || ')' - FROM airports - WHERE icao_code IS NOT NULL AND icao_code != '' - ORDER BY name - LIMIT 3 - `) - if err == nil { - for rows.Next() { - var airport string - if rows.Scan(&airport) == nil { - sampleAirports = append(sampleAirports, airport) - } - } - rows.Close() - if len(sampleAirports) > 0 { - fmt.Printf(" ā€¢ Sample Airports: %s\n", strings.Join(sampleAirports, ", ")) - } - } - } - - return nil -} - -// cmdUpdate updates data from specified sources (or safe sources by default) -func cmdUpdate(db *database.Database, sources []string, force bool) error { - availableSources := database.GetAvailableDataSources() - - // If no sources specified, use safe (non-consent-required) sources - if len(sources) == 0 { - log.Println("Updating from safe data sources...") - for _, source := range availableSources { - if !source.RequiresConsent { - sources = append(sources, strings.ToLower(strings.ReplaceAll(source.Name, " ", ""))) - } - } - - if len(sources) == 0 { - log.Println("No safe data sources available for automatic update") - return nil - } - log.Printf("Found %d safe data sources to update", len(sources)) - } - - loader := database.NewDataLoader(db.GetConnection()) - - for _, sourceName := range sources { - // Find matching source - var matchedSource *database.DataSource - for _, available := range availableSources { - if strings.EqualFold(strings.ReplaceAll(available.Name, " ", ""), sourceName) { - matchedSource = &available - break - } - } - - if matchedSource == nil { - log.Printf("āš ļø Unknown source: %s", sourceName) - continue - } - - // Check for consent requirement - if matchedSource.RequiresConsent && !force { - log.Printf("Skipping %s: requires license acceptance (%s)", matchedSource.Name, matchedSource.License) - log.Printf("Use --force to accept license terms, or 'skyview-data import %s' for interactive acceptance", sourceName) - continue - } - - // Set license acceptance for forced operations - if force && matchedSource.RequiresConsent { - matchedSource.UserAcceptedLicense = true - log.Printf("Accepting license terms for %s (%s)", matchedSource.Name, matchedSource.License) - } - - log.Printf("Loading %s...", matchedSource.Name) - - result, err := loader.LoadDataSource(*matchedSource) - if err != nil { - log.Printf("Failed to load %s: %v", matchedSource.Name, err) - continue - } - - log.Printf("Loaded %s: %d records in %v", matchedSource.Name, result.RecordsNew, result.Duration) - - if len(result.Errors) > 0 { - log.Printf(" %d errors occurred during import (first few):", len(result.Errors)) - for i, errMsg := range result.Errors { - if i >= 3 { break } - log.Printf(" %s", errMsg) - } - } - } - - log.Println("Update completed successfully") - return nil -} - -// cmdImport imports data from a specific source with interactive license acceptance -func cmdImport(db *database.Database, sourceName string, force bool) error { - availableSources := database.GetAvailableDataSources() - - var matchedSource *database.DataSource - for _, available := range availableSources { - if strings.EqualFold(strings.ReplaceAll(available.Name, " ", ""), sourceName) { - matchedSource = &available - break - } - } - - if matchedSource == nil { - return fmt.Errorf("unknown data source: %s", sourceName) - } - - // Handle license acceptance - if matchedSource.RequiresConsent && !force { - fmt.Printf("šŸ“„ License Information for %s\n", matchedSource.Name) - fmt.Printf(" License: %s\n", matchedSource.License) - fmt.Printf(" URL: %s\n", matchedSource.URL) - fmt.Println() - fmt.Printf("By importing this data, you agree to comply with the %s license terms.\n", matchedSource.License) - - if !askForConfirmation("Do you accept the license terms?") { - fmt.Println("Import cancelled.") - return nil - } - - matchedSource.UserAcceptedLicense = true - } - - fmt.Printf("šŸ“„ Importing %s...\n", matchedSource.Name) - - loader := database.NewDataLoader(db.GetConnection()) - result, err := loader.LoadDataSource(*matchedSource) - if err != nil { - return fmt.Errorf("import failed: %v", err) - } - - fmt.Printf("āœ… Import completed!\n") - fmt.Printf(" Records: %d loaded, %d errors\n", result.RecordsNew, result.RecordsError) - fmt.Printf(" Duration: %v\n", result.Duration) - - return nil -} - -// cmdClear removes data from a specific source -func cmdClear(db *database.Database, sourceName string, force bool) error { - if !force && !askForConfirmation(fmt.Sprintf("Clear all data from source '%s'?", sourceName)) { - fmt.Println("Operation cancelled.") - return nil - } - - loader := database.NewDataLoader(db.GetConnection()) - err := loader.ClearDataSource(sourceName) - if err != nil { - return fmt.Errorf("clear failed: %v", err) - } - - fmt.Printf("āœ… Cleared data from source: %s\n", sourceName) - return nil -} - -// cmdReset clears all data and resets the database -func cmdReset(db *database.Database, force bool) error { - if !force { - fmt.Println("āš ļø This will remove ALL data from the database!") - if !askForConfirmation("Are you sure you want to reset the database?") { - fmt.Println("Reset cancelled.") - return nil - } - } - - // This would require implementing a database reset function - fmt.Println("āŒ Database reset not yet implemented") - return fmt.Errorf("reset functionality not implemented") -} - -// askForConfirmation asks the user for yes/no confirmation -func askForConfirmation(question string) bool { - fmt.Printf("%s (y/N): ", question) - - var response string - fmt.Scanln(&response) - - response = strings.ToLower(strings.TrimSpace(response)) - return response == "y" || response == "yes" -} - -// loadConfig loads the shared configuration file -func loadConfig(configPath string) (*Config, error) { - data, err := os.ReadFile(configPath) - if err != nil { - return nil, fmt.Errorf("failed to read config file %s: %w", configPath, err) - } - - var config Config - if err := json.Unmarshal(data, &config); err != nil { - return nil, fmt.Errorf("failed to parse config file %s: %w", configPath, err) - } - - return &config, nil -} - -// initDatabaseFromConfig initializes database using shared configuration -func initDatabaseFromConfig(config *Config, dbPathOverride string) (*database.Database, error) { - var dbConfig *database.Config - - if config.Database != nil { - dbConfig = config.Database - } else { - dbConfig = database.DefaultConfig() - } - - // Allow command-line override of database path - if dbPathOverride != "" { - dbConfig.Path = dbPathOverride - } - - // Resolve database path if empty - if dbConfig.Path == "" { - resolvedPath, err := database.ResolveDatabasePath(dbConfig.Path) - if err != nil { - return nil, fmt.Errorf("failed to resolve database path: %w", err) - } - dbConfig.Path = resolvedPath - } - - // Create and initialize database - db, err := database.NewDatabase(dbConfig) - if err != nil { - return nil, fmt.Errorf("failed to create database: %w", err) - } - - if err := db.Initialize(); err != nil { - db.Close() - return nil, fmt.Errorf("failed to initialize database: %w", err) - } - - return db, nil -} - -// cmdOptimize optimizes the database for storage efficiency -func cmdOptimize(db *database.Database, force bool) error { - fmt.Println("Database Storage Optimization") - fmt.Println("============================") - - // We need to get the database path from the config - // For now, let's create a simple optimization manager - config := &database.Config{ - Path: "./dev-skyview.db", // Default path - this should be configurable - } - - // Create optimization manager - optimizer := database.NewOptimizationManager(db, config) - - // Get current stats - fmt.Println("šŸ“Š Current Database Statistics:") - stats, err := optimizer.GetOptimizationStats() - if err != nil { - return fmt.Errorf("failed to get database stats: %w", err) - } - - fmt.Printf(" ā€¢ Size: %.1f MB\n", float64(stats.DatabaseSize)/(1024*1024)) - fmt.Printf(" ā€¢ Page Size: %d bytes\n", stats.PageSize) - fmt.Printf(" ā€¢ Total Pages: %d\n", stats.PageCount) - fmt.Printf(" ā€¢ Used Pages: %d\n", stats.UsedPages) - fmt.Printf(" ā€¢ Free Pages: %d\n", stats.FreePages) - fmt.Printf(" ā€¢ Efficiency: %.1f%%\n", stats.Efficiency) - fmt.Printf(" ā€¢ Auto VACUUM: %v\n", stats.AutoVacuumEnabled) - - // Check if optimization is needed - needsOptimization := stats.FreePages > 0 || stats.Efficiency < 95.0 - - if !needsOptimization && !force { - fmt.Println("āœ… Database is already well optimized!") - fmt.Println(" Use --force to run optimization anyway") - return nil - } - - // Perform optimizations - if force && !needsOptimization { - fmt.Println("\nšŸ”§ Force optimization requested:") - } else { - fmt.Println("\nšŸ”§ Applying Optimizations:") - } - - if err := optimizer.VacuumDatabase(); err != nil { - return fmt.Errorf("VACUUM failed: %w", err) - } - - if err := optimizer.OptimizeDatabase(); err != nil { - return fmt.Errorf("optimization failed: %w", err) - } - - // Show final stats - fmt.Println("\nšŸ“ˆ Final Statistics:") - finalStats, err := optimizer.GetOptimizationStats() - if err != nil { - return fmt.Errorf("failed to get final stats: %w", err) - } - - fmt.Printf(" ā€¢ Size: %.1f MB\n", float64(finalStats.DatabaseSize)/(1024*1024)) - fmt.Printf(" ā€¢ Efficiency: %.1f%%\n", finalStats.Efficiency) - fmt.Printf(" ā€¢ Free Pages: %d\n", finalStats.FreePages) - - if stats.DatabaseSize > finalStats.DatabaseSize { - saved := stats.DatabaseSize - finalStats.DatabaseSize - fmt.Printf(" ā€¢ Space Saved: %.1f MB\n", float64(saved)/(1024*1024)) - } - - fmt.Println("\nāœ… Database optimization completed!") - return nil -} - diff --git a/config.example.json b/config.example.json index bbdb433..f275347 100644 --- a/config.example.json +++ b/config.example.json @@ -44,31 +44,5 @@ "history_limit": 1000, "stale_timeout": 60, "update_rate": 1 - }, - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } } } \ No newline at end of file diff --git a/debian/DEBIAN/postinst b/debian/DEBIAN/postinst index a686dfc..7288202 100755 --- a/debian/DEBIAN/postinst +++ b/debian/DEBIAN/postinst @@ -18,21 +18,6 @@ case "$1" in chown skyview-adsb:skyview-adsb /var/lib/skyview-adsb /var/log/skyview-adsb >/dev/null 2>&1 || true chmod 755 /var/lib/skyview-adsb /var/log/skyview-adsb >/dev/null 2>&1 || true - # Create database directory for skyview user (not skyview-adsb) - mkdir -p /var/lib/skyview >/dev/null 2>&1 || true - if getent passwd skyview >/dev/null 2>&1; then - chown skyview:skyview /var/lib/skyview >/dev/null 2>&1 || true - else - # Create skyview user for database management - if ! getent group skyview >/dev/null 2>&1; then - addgroup --system --quiet skyview - fi - adduser --system --ingroup skyview --home /var/lib/skyview \ - --no-create-home --disabled-password --shell /bin/false --quiet skyview - chown skyview:skyview /var/lib/skyview >/dev/null 2>&1 || true - fi - chmod 755 /var/lib/skyview >/dev/null 2>&1 || true - # Set permissions on config files if [ -f /etc/skyview-adsb/config.json ]; then chown root:skyview-adsb /etc/skyview-adsb/config.json >/dev/null 2>&1 || true @@ -40,33 +25,14 @@ case "$1" in fi - # Handle systemd services + # Handle systemd service systemctl daemon-reload >/dev/null 2>&1 || true - # Check if main service was previously enabled + # Check if service was previously enabled if systemctl is-enabled skyview-adsb >/dev/null 2>&1; then # Service was enabled, restart it systemctl restart skyview-adsb >/dev/null 2>&1 || true fi - - # Only restart database timer if it was already enabled - if systemctl is-enabled skyview-database-update.timer >/dev/null 2>&1; then - systemctl restart skyview-database-update.timer >/dev/null 2>&1 || true - fi - - # Initialize database on first install (but don't auto-enable timer) - if [ ! -f /var/lib/skyview/skyview.db ]; then - echo "Initializing SkyView database..." - sudo -u skyview /usr/bin/skyview-data update >/dev/null 2>&1 || true - echo "Database initialized with safe (public domain) data." - echo "" - echo "To enable automatic weekly updates:" - echo " systemctl enable --now skyview-database-update.timer" - echo "" - echo "To import additional data sources:" - echo " skyview-data list" - echo " skyview-data import " - fi ;; esac diff --git a/debian/lib/systemd/system/skyview-adsb.service b/debian/lib/systemd/system/skyview-adsb.service index faa3f50..c02c5fb 100644 --- a/debian/lib/systemd/system/skyview-adsb.service +++ b/debian/lib/systemd/system/skyview-adsb.service @@ -8,9 +8,6 @@ Wants=network.target Type=simple User=skyview-adsb Group=skyview-adsb -# Update database before starting main service -ExecStartPre=/usr/bin/skyview-data -config /etc/skyview-adsb/config.json update -TimeoutStartSec=300 ExecStart=/usr/bin/skyview -config /etc/skyview-adsb/config.json WorkingDirectory=/var/lib/skyview-adsb StandardOutput=journal diff --git a/debian/lib/systemd/system/skyview-database-update.service b/debian/lib/systemd/system/skyview-database-update.service deleted file mode 100644 index 9ab59ef..0000000 --- a/debian/lib/systemd/system/skyview-database-update.service +++ /dev/null @@ -1,33 +0,0 @@ -[Unit] -Description=SkyView Database Update -Documentation=man:skyview-data(1) -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -User=skyview-adsb -Group=skyview-adsb -ExecStart=/usr/bin/skyview-data update -StandardOutput=journal -StandardError=journal - -# Security hardening -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/var/lib/skyview-adsb /tmp -ProtectKernelTunables=true -ProtectKernelModules=true -ProtectControlGroups=true -RestrictRealtime=true -RestrictSUIDSGID=true - -# Resource limits -MemoryMax=256M -TasksMax=50 -TimeoutStartSec=300 - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/debian/lib/systemd/system/skyview-database-update.timer b/debian/lib/systemd/system/skyview-database-update.timer deleted file mode 100644 index 652903a..0000000 --- a/debian/lib/systemd/system/skyview-database-update.timer +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=SkyView Database Update Timer -Documentation=man:skyview-data(1) -Requires=skyview-database-update.service - -[Timer] -# Run weekly on Sunday at 3 AM -OnCalendar=Sun 03:00 -# Randomize start time within 1 hour to avoid thundering herd -RandomizedDelaySec=3600 -# Start immediately if system was down during scheduled time -Persistent=true -# Don't start if system is on battery (laptops) -ConditionACPower=true - -[Install] -WantedBy=timers.target \ No newline at end of file diff --git a/debian/usr/share/doc/skyview-adsb/CONFIGURATION.md b/debian/usr/share/doc/skyview-adsb/CONFIGURATION.md deleted file mode 100644 index f340aca..0000000 --- a/debian/usr/share/doc/skyview-adsb/CONFIGURATION.md +++ /dev/null @@ -1,708 +0,0 @@ -# SkyView Configuration Guide - -This document provides comprehensive configuration options for SkyView, including server settings, data sources, database management, and external aviation data integration. - -## Configuration File Format - -SkyView uses JSON configuration files. The default locations are: -- **System service**: `/etc/skyview-adsb/config.json` -- **User mode**: `~/.config/skyview/config.json` -- **Current directory**: `./config.json` -- **Custom path**: Specify with `-config path/to/config.json` - -## Complete Configuration Example - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "primary", - "name": "Primary Site", - "host": "localhost", - "port": 30005, - "format": "beast", - "latitude": 51.4700, - "longitude": -0.4600, - "altitude": 50.0, - "enabled": true - } - ], - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Control Tower" - }, - "settings": { - "history_limit": 1000, - "stale_timeout": 60, - "update_rate": 1 - }, - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } - } -} -``` - -## Configuration Sections - -### Server Configuration - -Controls the web server and API endpoints. - -```json -{ - "server": { - "host": "", - "port": 8080 - } -} -``` - -#### Options - -- **`host`** (string): Interface to bind to - - `""` or `"0.0.0.0"` = All interfaces (default) - - `"127.0.0.1"` = Localhost only (IPv4) - - `"::1"` = Localhost only (IPv6) - - `"192.168.1.100"` = Specific interface - -- **`port`** (integer): TCP port for web interface - - Default: `8080` - - Valid range: `1-65535` - - Ports below 1024 require root privileges - -### ADS-B Data Sources - -Configures connections to dump1090/readsb receivers. - -```json -{ - "sources": [ - { - "id": "primary", - "name": "Primary Site", - "host": "localhost", - "port": 30005, - "format": "beast", - "latitude": 51.4700, - "longitude": -0.4600, - "altitude": 50.0, - "enabled": true - } - ] -} -``` - -#### Source Options - -- **`id`** (string, required): Unique identifier for this source -- **`name`** (string, required): Human-readable name displayed in UI -- **`host`** (string, required): Hostname or IP address of receiver -- **`port`** (integer, required): TCP port of receiver -- **`format`** (string, optional): Data format - - `"beast"` = Beast binary format (port 30005, default) - - `"vrs"` = VRS JSON format (port 33005) -- **`latitude`** (number, required): Receiver latitude in decimal degrees -- **`longitude`** (number, required): Receiver longitude in decimal degrees -- **`altitude`** (number, optional): Receiver altitude in meters above sea level -- **`enabled`** (boolean): Enable/disable this source (default: `true`) - -#### Multiple Sources Example - -```json -{ - "sources": [ - { - "id": "site1", - "name": "North Site", - "host": "192.168.1.100", - "port": 30005, - "latitude": 51.50, - "longitude": -0.46, - "enabled": true - }, - { - "id": "site2", - "name": "South Site (VRS)", - "host": "192.168.1.101", - "port": 33005, - "format": "vrs", - "latitude": 51.44, - "longitude": -0.46, - "enabled": true - } - ] -} -``` - -### Map Origin Configuration - -Sets the default map center and reference point for the web interface. This is **different** from the ADS-B receiver locations defined in `sources`. - -```json -{ - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Control Tower" - } -} -``` - -#### Purpose and Usage - -The **`origin`** defines where the map centers when users first load the web interface: - -- **Map Center**: Initial view focus when loading the web interface -- **Reference Point**: Visual "home" location for navigation -- **User Experience**: Where operators expect to see coverage area -- **Reset Target**: Where "Reset to Origin" button returns the map view - -This is **separate from** the `sources` coordinates, which define: -- Physical ADS-B receiver locations for signal processing -- Multi-source data fusion calculations -- Coverage area computation -- Signal strength weighting - -#### Options - -- **`latitude`** (number, required): Center latitude in decimal degrees -- **`longitude`** (number, required): Center longitude in decimal degrees -- **`name`** (string, optional): Display name for the origin point (shown in UI) - -#### Example: Multi-Site Configuration - -```json -{ - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "London Control Center" - }, - "sources": [ - { - "id": "north", - "name": "North Site", - "latitude": 51.5200, - "longitude": -0.4600 - }, - { - "id": "south", - "name": "South Site", - "latitude": 51.4200, - "longitude": -0.4600 - } - ] -} -``` - -In this configuration: -- **Map centers** on the Control Center for optimal viewing -- **Two receivers** located north and south provide coverage -- **Users see** the control area as the focal point -- **System uses** both receiver locations for signal processing - -#### Single-Site Simplification - -For single receiver deployments, origin and source coordinates are typically the same: - -```json -{ - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Primary Site" - }, - "sources": [ - { - "id": "primary", - "name": "Primary Receiver", - "latitude": 51.4700, - "longitude": -0.4600 - } - ] -} -``` - -### General Settings - -Global application behavior settings. - -```json -{ - "settings": { - "history_limit": 1000, - "stale_timeout": 60, - "update_rate": 1 - } -} -``` - -#### Options - -- **`history_limit`** (integer): Maximum aircraft position history points per aircraft - - Default: `1000` - - Higher values = longer trails, more memory usage - - `0` = No history limit - -- **`stale_timeout`** (integer): Seconds before aircraft is considered stale/inactive - - Default: `60` seconds - - Range: `10-600` seconds - -- **`update_rate`** (integer): WebSocket update rate in seconds - - Default: `1` second - - Range: `1-10` seconds - - Lower values = more responsive, higher bandwidth - -### Database Configuration - -Controls SQLite database storage and performance. - -```json -{ - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - } -} -``` - -#### Options - -- **`path`** (string): Database file path - - `""` = Auto-resolve to system appropriate location - - Custom path: `"/var/lib/skyview-adsb/skyview.db"` - - Relative path: `"./data/skyview.db"` - -- **`max_history_days`** (integer): Aircraft history retention in days - - Default: `7` days - - `0` = Keep all history (unlimited) - - Range: `1-365` days - -- **`backup_on_upgrade`** (boolean): Create backup before schema upgrades - - Default: `true` - - Recommended to keep enabled for safety - -- **`max_open_conns`** (integer): Maximum concurrent database connections - - Default: `10` - - Range: `1-100` - -- **`max_idle_conns`** (integer): Maximum idle database connections in pool - - Default: `5` - - Range: `1-50` - - Should be ā‰¤ max_open_conns - -## External Aviation Data Sources - -SkyView can enhance aircraft data using external aviation databases. This section configures callsign enhancement and airline/airport lookup functionality. - -### Callsign Enhancement Configuration - -```json -{ - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } - } -} -``` - -#### Main Callsign Options - -- **`enabled`** (boolean): Enable callsign enhancement features - - Default: `true` - - Set to `false` to disable all callsign lookups - -- **`cache_hours`** (integer): Hours to cache lookup results - - Default: `24` hours - - Range: `1-168` hours (1 hour to 1 week) - -- **`privacy_mode`** (boolean): Disable all external data requests - - Default: `false` - - `true` = Local-only operation (no network requests) - - `false` = Allow external data loading - -### Available External Data Sources - -SkyView supports three external aviation data sources: - -#### 1. OpenFlights Airlines Database - -- **Content**: Global airline information with ICAO/IATA codes, callsigns, and country data -- **Records**: ~6,162 airlines worldwide -- **License**: AGPL-3.0 (runtime consumption allowed) -- **Source**: https://openflights.org/data.html -- **Updates**: Downloads latest data automatically - -```json -"openflights_airlines": { - "enabled": true, - "priority": 1 -} -``` - -#### 2. OpenFlights Airports Database - -- **Content**: Global airport data with coordinates, codes, and basic metadata -- **Records**: ~7,698 airports worldwide -- **License**: AGPL-3.0 (runtime consumption allowed) -- **Source**: https://openflights.org/data.html -- **Updates**: Downloads latest data automatically - -```json -"openflights_airports": { - "enabled": true, - "priority": 2 -} -``` - -#### 3. OurAirports Database - -- **Content**: Comprehensive airport database with detailed geographic and operational metadata -- **Records**: ~83,557 airports worldwide (includes small airfields) -- **License**: Public Domain (CC0) -- **Source**: https://ourairports.com/data/ -- **Updates**: Downloads latest data automatically - -```json -"ourairports": { - "enabled": true, - "priority": 3 -} -``` - -#### Source Configuration Options - -- **`enabled`** (boolean): Enable/disable this specific data source -- **`priority`** (integer): Processing priority (lower number = higher priority) - -**Note**: License information and consent requirements are handled automatically by the system. All currently available data sources are safe for automatic loading without explicit consent. - -### Data Loading Performance - -When all sources are enabled, expect the following performance: - -- **OpenFlights Airlines**: 6,162 records in ~350ms -- **OpenFlights Airports**: 7,698 records in ~640ms -- **OurAirports**: 83,557 records in ~2.2s -- **Total**: 97,417 records in ~3.2s - -## Privacy and Security Settings - -### Privacy Mode - -Enable privacy mode to disable all external data requests: - -```json -{ - "callsign": { - "privacy_mode": true, - "sources": { - "openflights_airlines": { - "enabled": false - }, - "openflights_airports": { - "enabled": false - }, - "ourairports": { - "enabled": false - } - } - } -} -``` - -#### Privacy Mode Features - -- **No External Requests**: Completely disables all external data loading -- **Local-Only Operation**: Uses only embedded data and local cache -- **Air-Gapped Compatible**: Suitable for isolated networks -- **Compliance**: Meets strict privacy requirements - -### Selective Source Control - -Enable only specific data sources: - -```json -{ - "callsign": { - "sources": { - "openflights_airlines": { - "enabled": true - }, - "openflights_airports": { - "enabled": false - }, - "ourairports": { - "enabled": false - } - } - } -} -``` - -## Database Management Commands - -### Updating External Data Sources - -```bash -# Update all enabled external data sources -skyview-data -config /path/to/config.json update - -# List available data sources -skyview-data -config /path/to/config.json list - -# Check database status and loaded sources -skyview-data -config /path/to/config.json status -``` - -### Database Optimization - -```bash -# Optimize database storage efficiency -skyview-data -config /path/to/config.json optimize - -# Check optimization statistics only -skyview-data -config /path/to/config.json optimize --stats-only - -# Force optimization without prompts -skyview-data -config /path/to/config.json optimize --force -``` - -## Configuration Validation - -### Validating Configuration - -```bash -# Test configuration file syntax -skyview -config config.json -test - -# Verify data source connectivity -skyview-data -config config.json list -``` - -### Common Configuration Errors - -#### JSON Syntax Errors -``` -Error: invalid character '}' looking for beginning of object key string -``` -**Solution**: Check for trailing commas, missing quotes, or bracket mismatches. - -#### Invalid Data Types -``` -Error: json: cannot unmarshal string into Go struct field -``` -**Solution**: Ensure numbers are not quoted, booleans use true/false, etc. - -#### Missing Required Fields -``` -Error: source 'primary' missing required field: latitude -``` -**Solution**: Add all required fields for each configured source. - -### Minimal Configuration - -For basic operation without external data sources: - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "primary", - "name": "Local Receiver", - "host": "localhost", - "port": 30005, - "latitude": 51.4700, - "longitude": -0.4600, - "enabled": true - } - ], - "settings": { - "history_limit": 1000, - "stale_timeout": 60, - "update_rate": 1 - } -} -``` - -### Full-Featured Configuration - -For complete functionality with all external data sources: - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "primary", - "name": "Primary Site", - "host": "localhost", - "port": 30005, - "latitude": 51.4700, - "longitude": -0.4600, - "altitude": 50.0, - "enabled": true - } - ], - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Control Tower" - }, - "settings": { - "history_limit": 4000, - "stale_timeout": 60, - "update_rate": 1 - }, - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } - } -} -``` - -This configuration enables all SkyView features including multi-source ADS-B data fusion, comprehensive aviation database integration, and database optimization. - -## Environment-Specific Examples - -### Production System Service - -Configuration for systemd service deployment: - -```json -{ - "server": { - "host": "0.0.0.0", - "port": 8080 - }, - "database": { - "path": "/var/lib/skyview-adsb/skyview.db", - "max_history_days": 30, - "backup_on_upgrade": true - }, - "callsign": { - "enabled": true, - "privacy_mode": false - } -} -``` - -### Development/Testing - -Configuration for development use: - -```json -{ - "server": { - "host": "127.0.0.1", - "port": 3000 - }, - "database": { - "path": "./dev-skyview.db", - "max_history_days": 1 - }, - "settings": { - "history_limit": 100, - "update_rate": 1 - } -} -``` - -### Air-Gapped/Secure Environment - -Configuration for isolated networks: - -```json -{ - "callsign": { - "enabled": true, - "privacy_mode": true, - "sources": { - "openflights_airlines": { - "enabled": false - }, - "openflights_airports": { - "enabled": false - }, - "ourairports": { - "enabled": false - } - } - } -} -``` \ No newline at end of file diff --git a/debian/usr/share/doc/skyview-adsb/DATABASE.md b/debian/usr/share/doc/skyview-adsb/DATABASE.md deleted file mode 100644 index 2e7347d..0000000 --- a/debian/usr/share/doc/skyview-adsb/DATABASE.md +++ /dev/null @@ -1,99 +0,0 @@ -# SkyView Database Management - -SkyView includes a comprehensive database management system for enriching aircraft callsigns with airline and airport information. - -## Quick Start - -### 1. Check Current Status -```bash -skyview-data status -``` - -### 2. Import Safe Data (Recommended) -```bash -# Import public domain sources automatically -skyview-data update -``` - -### 3. Enable Automatic Updates (Optional) -```bash -# Weekly updates on Sunday at 3 AM -sudo systemctl enable --now skyview-database-update.timer -``` - -## Available Data Sources - -### Safe Sources (Public Domain) -These sources are imported automatically with `skyview-data update`: -- **OurAirports**: Comprehensive airport database (public domain) -- **FAA Registry**: US aircraft registration data (public domain) - -### License-Required Sources -These require explicit acceptance: -- **OpenFlights**: Airline and airport data (AGPL-3.0 license) - -## Commands - -### Basic Operations -```bash -skyview-data list # Show available sources -skyview-data status # Show database status -skyview-data update # Update safe sources -skyview-data import openflights # Import licensed source -skyview-data clear # Remove source data -``` - -### Systemd Timer Management -```bash -# Enable weekly automatic updates -systemctl enable skyview-database-update.timer -systemctl start skyview-database-update.timer - -# Check timer status -systemctl status skyview-database-update.timer - -# View update logs -journalctl -u skyview-database-update.service - -# Disable automatic updates -systemctl disable skyview-database-update.timer -``` - -## License Compliance - -SkyView maintains strict license separation: -- **SkyView binary**: Contains no external data (stays MIT licensed) -- **Runtime import**: Users choose which sources to import -- **Safe defaults**: Only public domain sources updated automatically -- **User choice**: Each person decides their own license compatibility - -## Troubleshooting - -### Check Service Status -```bash -systemctl status skyview-database-update.timer -journalctl -u skyview-database-update.service -f -``` - -### Manual Database Reset -```bash -systemctl stop skyview-database-update.timer -skyview-data reset --force -skyview-data update -systemctl start skyview-database-update.timer -``` - -### Permissions Issues -```bash -sudo chown skyview:skyview /var/lib/skyview/ -sudo chmod 755 /var/lib/skyview/ -``` - -## Files and Directories - -- `/usr/bin/skyview-data` - Database management command -- `/var/lib/skyview/skyview.db` - Database file -- `/usr/share/skyview/scripts/update-database.sh` - Cron helper script -- `/lib/systemd/system/skyview-database-update.*` - Systemd timer files - -For detailed information, see `man skyview-data`. \ No newline at end of file diff --git a/debian/usr/share/man/man1/skyview-data.1 b/debian/usr/share/man/man1/skyview-data.1 deleted file mode 100644 index 96c6540..0000000 --- a/debian/usr/share/man/man1/skyview-data.1 +++ /dev/null @@ -1,181 +0,0 @@ -.TH skyview-data 1 "January 2025" "SkyView Database Manager" -.SH NAME -skyview-data \- SkyView aviation database management utility - -.SH SYNOPSIS -.B skyview-data -[\fIOPTIONS\fR] \fICOMMAND\fR [\fIARGS\fR...] - -.SH DESCRIPTION -.B skyview-data -manages the SkyView aviation database, allowing users to import airline and airport data from various external sources while maintaining license compliance. - -The tool automatically creates and migrates the database schema, downloads data from public and licensed sources, and provides status monitoring for the aviation database used by SkyView for callsign enhancement. - -.SH OPTIONS -.TP -.BR \-db " \fIPATH\fR" -Database file path (auto-detected if empty) -.TP -.BR \-v ", " \-\-verbose -Verbose output -.TP -.BR \-\-force -Force operation without prompts -.TP -.BR \-\-version -Show version information - -.SH COMMANDS -.TP -.B init -Initialize empty database with schema -.TP -.B list -List available data sources with license information -.TP -.B status -Show current database status and statistics -.TP -.B update [\fISOURCE\fR...] -Update data from specified sources, or all safe sources if none specified -.TP -.B import \fISOURCE\fR -Import data from a specific source with license acceptance -.TP -.B clear \fISOURCE\fR -Remove all data from the specified source -.TP -.B reset -Clear all data and reset database (destructive) - -.SH DATA SOURCES -.SS Safe Sources (Public Domain) -These sources are automatically included in -.B update -operations: -.TP -.B ourairports -Public domain airport database from OurAirports.com -.TP -.B faa-registry -US aircraft registration database (FAA, public domain) - -.SS License-Required Sources -These sources require explicit license acceptance: -.TP -.B openflights -Comprehensive airline and airport database (AGPL-3.0 license) - -.SH EXAMPLES -.TP -Initialize database and import safe data: -.EX -skyview-data init -skyview-data update -.EE -.TP -Import OpenFlights data with license acceptance: -.EX -skyview-data import openflights -.EE -.TP -Check database status: -.EX -skyview-data status -.EE -.TP -Set up automatic updates via systemd timer: -.EX -systemctl enable skyview-database-update.timer -systemctl start skyview-database-update.timer -.EE - -.SH CRON AUTOMATION -For automated updates, -.B skyview-data update -is designed to work seamlessly with cron: - -.EX -# Update weekly on Sunday at 3 AM -0 3 * * 0 /usr/bin/skyview-data update -.EE - -The command automatically: -.RS -.IP \(bu 2 -Creates the database if it doesn't exist -.IP \(bu 2 -Updates only safe (public domain) sources -.IP \(bu 2 -Provides proper exit codes for monitoring -.IP \(bu 2 -Logs to standard output with timestamps -.RE - -.SH SYSTEMD INTEGRATION -The Debian package includes systemd timer integration: - -.EX -# Enable automatic weekly updates -systemctl enable skyview-database-update.timer -systemctl start skyview-database-update.timer - -# Check timer status -systemctl status skyview-database-update.timer - -# View update logs -journalctl -u skyview-database-update.service -.EE - -.SH FILES -.TP -.I /var/lib/skyview/skyview.db -System-wide database location -.TP -.I ~/.local/share/skyview/skyview.db -User-specific database location -.TP -.I /var/log/skyview/ -Log directory for database operations - -.SH EXIT STATUS -.TP -.B 0 -Success -.TP -.B 1 -General error or command failure -.TP -.B 2 -Invalid arguments or usage - -.SH SECURITY -All external data downloads use HTTPS. No sensitive information is transmitted. The tool processes only publicly available aviation data. - -License-required sources require explicit user acceptance before import. - -.SH LICENSE COMPLIANCE -.B skyview-data -maintains strict license separation: -.RS -.IP \(bu 2 -SkyView binary contains no external data (MIT license maintained) -.IP \(bu 2 -Each data source tracks its license and user acceptance -.IP \(bu 2 -Users choose which sources to import based on license compatibility -.IP \(bu 2 -Automatic updates only include public domain sources -.RE - -.SH SEE ALSO -.BR skyview (1), -.BR systemctl (1), -.BR crontab (5) - -.SH AUTHOR -SkyView is developed as an open-source ADS-B aircraft tracking system. - -.SH REPORTING BUGS -Report bugs and issues at the project repository. \ No newline at end of file diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 940b23c..7a93420 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -107,35 +107,11 @@ SkyView is a high-performance, multi-source ADS-B aircraft tracking system built - Aircraft state management and lifecycle tracking - Historical data collection (position, altitude, speed, signal trails) - Automatic stale aircraft cleanup -- SQLite database integration for persistent storage **Files**: - `merger.go`: Multi-source data fusion engine -### 4. Database System (`internal/database/`) - -**Purpose**: Provides persistent storage for historical aircraft data and callsign enhancement - -**Key Features**: -- SQLite-based storage with versioned schema migrations -- Aircraft position history with configurable retention -- Embedded OpenFlights airline and airport databases -- Callsign lookup cache for external API results -- Privacy mode for air-gapped operation -- Automatic database maintenance and cleanup - -**Files**: -- `database.go`: Core database operations and schema management -- `migrations.go`: Database schema versioning and migration system -- `callsign.go`: Callsign enhancement and cache management - -**Storage Components**: -- **Aircraft History**: Time-series position data with source attribution -- **OpenFlights Data**: Embedded airline/airport reference data -- **Callsign Cache**: External API lookup results with TTL -- **Schema Versioning**: Migration tracking and rollback support - -### 5. ICAO Country Database (`internal/icao/`) +### 4. ICAO Country Database (`internal/icao/`) **Purpose**: Provides comprehensive ICAO address to country mapping @@ -149,7 +125,7 @@ SkyView is a high-performance, multi-source ADS-B aircraft tracking system built **Files**: - `database.go`: In-memory ICAO allocation database with binary search -### 6. HTTP/WebSocket Server (`internal/server/`) +### 5. HTTP/WebSocket Server (`internal/server/`) **Purpose**: Serves web interface and provides low-latency data streaming @@ -162,7 +138,7 @@ SkyView is a high-performance, multi-source ADS-B aircraft tracking system built **Files**: - `server.go`: HTTP server and WebSocket handler -### 7. Web Frontend (`assets/static/`) +### 6. Web Frontend (`assets/static/`) **Purpose**: Interactive web interface for aircraft tracking and visualization @@ -244,17 +220,6 @@ SkyView is a high-performance, multi-source ADS-B aircraft tracking system built "latitude": 51.4700, // Map center point "longitude": -0.4600, "name": "Origin Name" - }, - "database": { - "path": "", // Auto-resolved: /var/lib/skyview/skyview.db - "max_history_days": 7, // Data retention period - "backup_on_upgrade": true // Backup before migrations - }, - "callsign": { - "enabled": true, // Enable callsign enhancement - "cache_hours": 24, // External API cache TTL - "external_apis": true, // Allow external API calls - "privacy_mode": false // Disable external data transmission } } ``` @@ -268,8 +233,7 @@ SkyView is a high-performance, multi-source ADS-B aircraft tracking system built - **Non-blocking I/O**: Asynchronous network operations ### Memory Management -- **Database Storage**: Persistent history reduces memory usage -- **Configurable Retention**: Database cleanup based on age and limits +- **Bounded History**: Configurable limits on historical data storage - **Automatic Cleanup**: Stale aircraft removal to prevent memory leaks - **Efficient Data Structures**: Maps for O(1) aircraft lookups - **Embedded Assets**: Static files bundled in binary @@ -296,8 +260,7 @@ SkyView is a high-performance, multi-source ADS-B aircraft tracking system built ### Data Privacy - **Public ADS-B Data**: Only processes publicly broadcast aircraft data - **No Personal Information**: Aircraft tracking only, no passenger data -- **Privacy Mode**: Complete offline operation with external API disable -- **Local Processing**: All data processed and stored locally +- **Local Processing**: No data transmitted to external services - **Historical Limits**: Configurable data retention periods ## External Resources diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md deleted file mode 100644 index f340aca..0000000 --- a/docs/CONFIGURATION.md +++ /dev/null @@ -1,708 +0,0 @@ -# SkyView Configuration Guide - -This document provides comprehensive configuration options for SkyView, including server settings, data sources, database management, and external aviation data integration. - -## Configuration File Format - -SkyView uses JSON configuration files. The default locations are: -- **System service**: `/etc/skyview-adsb/config.json` -- **User mode**: `~/.config/skyview/config.json` -- **Current directory**: `./config.json` -- **Custom path**: Specify with `-config path/to/config.json` - -## Complete Configuration Example - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "primary", - "name": "Primary Site", - "host": "localhost", - "port": 30005, - "format": "beast", - "latitude": 51.4700, - "longitude": -0.4600, - "altitude": 50.0, - "enabled": true - } - ], - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Control Tower" - }, - "settings": { - "history_limit": 1000, - "stale_timeout": 60, - "update_rate": 1 - }, - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } - } -} -``` - -## Configuration Sections - -### Server Configuration - -Controls the web server and API endpoints. - -```json -{ - "server": { - "host": "", - "port": 8080 - } -} -``` - -#### Options - -- **`host`** (string): Interface to bind to - - `""` or `"0.0.0.0"` = All interfaces (default) - - `"127.0.0.1"` = Localhost only (IPv4) - - `"::1"` = Localhost only (IPv6) - - `"192.168.1.100"` = Specific interface - -- **`port`** (integer): TCP port for web interface - - Default: `8080` - - Valid range: `1-65535` - - Ports below 1024 require root privileges - -### ADS-B Data Sources - -Configures connections to dump1090/readsb receivers. - -```json -{ - "sources": [ - { - "id": "primary", - "name": "Primary Site", - "host": "localhost", - "port": 30005, - "format": "beast", - "latitude": 51.4700, - "longitude": -0.4600, - "altitude": 50.0, - "enabled": true - } - ] -} -``` - -#### Source Options - -- **`id`** (string, required): Unique identifier for this source -- **`name`** (string, required): Human-readable name displayed in UI -- **`host`** (string, required): Hostname or IP address of receiver -- **`port`** (integer, required): TCP port of receiver -- **`format`** (string, optional): Data format - - `"beast"` = Beast binary format (port 30005, default) - - `"vrs"` = VRS JSON format (port 33005) -- **`latitude`** (number, required): Receiver latitude in decimal degrees -- **`longitude`** (number, required): Receiver longitude in decimal degrees -- **`altitude`** (number, optional): Receiver altitude in meters above sea level -- **`enabled`** (boolean): Enable/disable this source (default: `true`) - -#### Multiple Sources Example - -```json -{ - "sources": [ - { - "id": "site1", - "name": "North Site", - "host": "192.168.1.100", - "port": 30005, - "latitude": 51.50, - "longitude": -0.46, - "enabled": true - }, - { - "id": "site2", - "name": "South Site (VRS)", - "host": "192.168.1.101", - "port": 33005, - "format": "vrs", - "latitude": 51.44, - "longitude": -0.46, - "enabled": true - } - ] -} -``` - -### Map Origin Configuration - -Sets the default map center and reference point for the web interface. This is **different** from the ADS-B receiver locations defined in `sources`. - -```json -{ - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Control Tower" - } -} -``` - -#### Purpose and Usage - -The **`origin`** defines where the map centers when users first load the web interface: - -- **Map Center**: Initial view focus when loading the web interface -- **Reference Point**: Visual "home" location for navigation -- **User Experience**: Where operators expect to see coverage area -- **Reset Target**: Where "Reset to Origin" button returns the map view - -This is **separate from** the `sources` coordinates, which define: -- Physical ADS-B receiver locations for signal processing -- Multi-source data fusion calculations -- Coverage area computation -- Signal strength weighting - -#### Options - -- **`latitude`** (number, required): Center latitude in decimal degrees -- **`longitude`** (number, required): Center longitude in decimal degrees -- **`name`** (string, optional): Display name for the origin point (shown in UI) - -#### Example: Multi-Site Configuration - -```json -{ - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "London Control Center" - }, - "sources": [ - { - "id": "north", - "name": "North Site", - "latitude": 51.5200, - "longitude": -0.4600 - }, - { - "id": "south", - "name": "South Site", - "latitude": 51.4200, - "longitude": -0.4600 - } - ] -} -``` - -In this configuration: -- **Map centers** on the Control Center for optimal viewing -- **Two receivers** located north and south provide coverage -- **Users see** the control area as the focal point -- **System uses** both receiver locations for signal processing - -#### Single-Site Simplification - -For single receiver deployments, origin and source coordinates are typically the same: - -```json -{ - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Primary Site" - }, - "sources": [ - { - "id": "primary", - "name": "Primary Receiver", - "latitude": 51.4700, - "longitude": -0.4600 - } - ] -} -``` - -### General Settings - -Global application behavior settings. - -```json -{ - "settings": { - "history_limit": 1000, - "stale_timeout": 60, - "update_rate": 1 - } -} -``` - -#### Options - -- **`history_limit`** (integer): Maximum aircraft position history points per aircraft - - Default: `1000` - - Higher values = longer trails, more memory usage - - `0` = No history limit - -- **`stale_timeout`** (integer): Seconds before aircraft is considered stale/inactive - - Default: `60` seconds - - Range: `10-600` seconds - -- **`update_rate`** (integer): WebSocket update rate in seconds - - Default: `1` second - - Range: `1-10` seconds - - Lower values = more responsive, higher bandwidth - -### Database Configuration - -Controls SQLite database storage and performance. - -```json -{ - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - } -} -``` - -#### Options - -- **`path`** (string): Database file path - - `""` = Auto-resolve to system appropriate location - - Custom path: `"/var/lib/skyview-adsb/skyview.db"` - - Relative path: `"./data/skyview.db"` - -- **`max_history_days`** (integer): Aircraft history retention in days - - Default: `7` days - - `0` = Keep all history (unlimited) - - Range: `1-365` days - -- **`backup_on_upgrade`** (boolean): Create backup before schema upgrades - - Default: `true` - - Recommended to keep enabled for safety - -- **`max_open_conns`** (integer): Maximum concurrent database connections - - Default: `10` - - Range: `1-100` - -- **`max_idle_conns`** (integer): Maximum idle database connections in pool - - Default: `5` - - Range: `1-50` - - Should be ā‰¤ max_open_conns - -## External Aviation Data Sources - -SkyView can enhance aircraft data using external aviation databases. This section configures callsign enhancement and airline/airport lookup functionality. - -### Callsign Enhancement Configuration - -```json -{ - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } - } -} -``` - -#### Main Callsign Options - -- **`enabled`** (boolean): Enable callsign enhancement features - - Default: `true` - - Set to `false` to disable all callsign lookups - -- **`cache_hours`** (integer): Hours to cache lookup results - - Default: `24` hours - - Range: `1-168` hours (1 hour to 1 week) - -- **`privacy_mode`** (boolean): Disable all external data requests - - Default: `false` - - `true` = Local-only operation (no network requests) - - `false` = Allow external data loading - -### Available External Data Sources - -SkyView supports three external aviation data sources: - -#### 1. OpenFlights Airlines Database - -- **Content**: Global airline information with ICAO/IATA codes, callsigns, and country data -- **Records**: ~6,162 airlines worldwide -- **License**: AGPL-3.0 (runtime consumption allowed) -- **Source**: https://openflights.org/data.html -- **Updates**: Downloads latest data automatically - -```json -"openflights_airlines": { - "enabled": true, - "priority": 1 -} -``` - -#### 2. OpenFlights Airports Database - -- **Content**: Global airport data with coordinates, codes, and basic metadata -- **Records**: ~7,698 airports worldwide -- **License**: AGPL-3.0 (runtime consumption allowed) -- **Source**: https://openflights.org/data.html -- **Updates**: Downloads latest data automatically - -```json -"openflights_airports": { - "enabled": true, - "priority": 2 -} -``` - -#### 3. OurAirports Database - -- **Content**: Comprehensive airport database with detailed geographic and operational metadata -- **Records**: ~83,557 airports worldwide (includes small airfields) -- **License**: Public Domain (CC0) -- **Source**: https://ourairports.com/data/ -- **Updates**: Downloads latest data automatically - -```json -"ourairports": { - "enabled": true, - "priority": 3 -} -``` - -#### Source Configuration Options - -- **`enabled`** (boolean): Enable/disable this specific data source -- **`priority`** (integer): Processing priority (lower number = higher priority) - -**Note**: License information and consent requirements are handled automatically by the system. All currently available data sources are safe for automatic loading without explicit consent. - -### Data Loading Performance - -When all sources are enabled, expect the following performance: - -- **OpenFlights Airlines**: 6,162 records in ~350ms -- **OpenFlights Airports**: 7,698 records in ~640ms -- **OurAirports**: 83,557 records in ~2.2s -- **Total**: 97,417 records in ~3.2s - -## Privacy and Security Settings - -### Privacy Mode - -Enable privacy mode to disable all external data requests: - -```json -{ - "callsign": { - "privacy_mode": true, - "sources": { - "openflights_airlines": { - "enabled": false - }, - "openflights_airports": { - "enabled": false - }, - "ourairports": { - "enabled": false - } - } - } -} -``` - -#### Privacy Mode Features - -- **No External Requests**: Completely disables all external data loading -- **Local-Only Operation**: Uses only embedded data and local cache -- **Air-Gapped Compatible**: Suitable for isolated networks -- **Compliance**: Meets strict privacy requirements - -### Selective Source Control - -Enable only specific data sources: - -```json -{ - "callsign": { - "sources": { - "openflights_airlines": { - "enabled": true - }, - "openflights_airports": { - "enabled": false - }, - "ourairports": { - "enabled": false - } - } - } -} -``` - -## Database Management Commands - -### Updating External Data Sources - -```bash -# Update all enabled external data sources -skyview-data -config /path/to/config.json update - -# List available data sources -skyview-data -config /path/to/config.json list - -# Check database status and loaded sources -skyview-data -config /path/to/config.json status -``` - -### Database Optimization - -```bash -# Optimize database storage efficiency -skyview-data -config /path/to/config.json optimize - -# Check optimization statistics only -skyview-data -config /path/to/config.json optimize --stats-only - -# Force optimization without prompts -skyview-data -config /path/to/config.json optimize --force -``` - -## Configuration Validation - -### Validating Configuration - -```bash -# Test configuration file syntax -skyview -config config.json -test - -# Verify data source connectivity -skyview-data -config config.json list -``` - -### Common Configuration Errors - -#### JSON Syntax Errors -``` -Error: invalid character '}' looking for beginning of object key string -``` -**Solution**: Check for trailing commas, missing quotes, or bracket mismatches. - -#### Invalid Data Types -``` -Error: json: cannot unmarshal string into Go struct field -``` -**Solution**: Ensure numbers are not quoted, booleans use true/false, etc. - -#### Missing Required Fields -``` -Error: source 'primary' missing required field: latitude -``` -**Solution**: Add all required fields for each configured source. - -### Minimal Configuration - -For basic operation without external data sources: - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "primary", - "name": "Local Receiver", - "host": "localhost", - "port": 30005, - "latitude": 51.4700, - "longitude": -0.4600, - "enabled": true - } - ], - "settings": { - "history_limit": 1000, - "stale_timeout": 60, - "update_rate": 1 - } -} -``` - -### Full-Featured Configuration - -For complete functionality with all external data sources: - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "primary", - "name": "Primary Site", - "host": "localhost", - "port": 30005, - "latitude": 51.4700, - "longitude": -0.4600, - "altitude": 50.0, - "enabled": true - } - ], - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Control Tower" - }, - "settings": { - "history_limit": 4000, - "stale_timeout": 60, - "update_rate": 1 - }, - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true, - "max_open_conns": 10, - "max_idle_conns": 5 - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_airlines": { - "enabled": true, - "priority": 1 - }, - "openflights_airports": { - "enabled": true, - "priority": 2 - }, - "ourairports": { - "enabled": true, - "priority": 3 - } - } - } -} -``` - -This configuration enables all SkyView features including multi-source ADS-B data fusion, comprehensive aviation database integration, and database optimization. - -## Environment-Specific Examples - -### Production System Service - -Configuration for systemd service deployment: - -```json -{ - "server": { - "host": "0.0.0.0", - "port": 8080 - }, - "database": { - "path": "/var/lib/skyview-adsb/skyview.db", - "max_history_days": 30, - "backup_on_upgrade": true - }, - "callsign": { - "enabled": true, - "privacy_mode": false - } -} -``` - -### Development/Testing - -Configuration for development use: - -```json -{ - "server": { - "host": "127.0.0.1", - "port": 3000 - }, - "database": { - "path": "./dev-skyview.db", - "max_history_days": 1 - }, - "settings": { - "history_limit": 100, - "update_rate": 1 - } -} -``` - -### Air-Gapped/Secure Environment - -Configuration for isolated networks: - -```json -{ - "callsign": { - "enabled": true, - "privacy_mode": true, - "sources": { - "openflights_airlines": { - "enabled": false - }, - "openflights_airports": { - "enabled": false - }, - "ourairports": { - "enabled": false - } - } - } -} -``` \ No newline at end of file diff --git a/docs/CRON_SETUP.md b/docs/CRON_SETUP.md deleted file mode 100644 index 7632d21..0000000 --- a/docs/CRON_SETUP.md +++ /dev/null @@ -1,259 +0,0 @@ -# SkyView Database Auto-Update with Cron - -This guide explains how to set up automatic database updates for SkyView using cron jobs. - -## Overview - -SkyView can automatically update its aviation database from public domain sources using cron jobs. This ensures your aircraft callsign data stays current without manual intervention. - -## Features - -- āœ… **Auto-initialization**: Creates empty database if it doesn't exist -- āœ… **Safe sources only**: Updates only public domain data (no license issues) -- āœ… **Cron-friendly**: Proper logging and exit codes for automated execution -- āœ… **Lock file protection**: Prevents concurrent updates -- āœ… **Error handling**: Graceful failure handling with logging - -## Quick Setup - -### 1. Command Line Tool - -The `skyview-data` command is designed to work perfectly with cron: - -```bash -# Auto-initialize database and update safe sources -skyview-data update - -# Check what's loaded -skyview-data status - -# List available sources -skyview-data list -``` - -### 2. Cron Job Examples - -#### Daily Update (Recommended) -```bash -# Add to crontab: crontab -e -# Update database daily at 3 AM -0 3 * * * /usr/bin/skyview-data update >/var/log/skyview-update.log 2>&1 -``` - -#### Weekly Update -```bash -# Update database weekly on Sunday at 2 AM -0 2 * * 0 /usr/bin/skyview-data update >/var/log/skyview-update.log 2>&1 -``` - -#### With Helper Script (Debian Package) -```bash -# Use the provided update script -0 3 * * * /usr/share/skyview/scripts/update-database.sh -``` - -### 3. System Service User - -For Debian package installations, use the skyview service user: - -```bash -# Edit skyview user's crontab -sudo crontab -u skyview -e - -# Add daily update -0 3 * * * /usr/bin/skyview-data update -``` - -## Configuration - -### Database Location - -The tool automatically detects the database location: -- **System service**: `/var/lib/skyview/skyview.db` -- **User install**: `~/.local/share/skyview/skyview.db` -- **Current directory**: `./skyview.db` - -### Custom Database Path - -```bash -# Specify custom database location -skyview-data -db /custom/path/skyview.db update -``` - -### Logging - -For cron jobs, redirect output to log files: - -```bash -# Basic logging -skyview-data update >> /var/log/skyview-update.log 2>&1 - -# With timestamps (using helper script) -/usr/share/skyview/scripts/update-database.sh -``` - -## Data Sources - -### Safe Sources (Auto-Updated) - -These sources are automatically included in `skyview-data update`: - -- **OurAirports**: Public domain airport data -- **FAA Registry**: US aircraft registration (public domain) -- *(Additional safe sources as they become available)* - -### License-Required Sources - -These require explicit acceptance and are NOT included in automatic updates: - -- **OpenFlights**: AGPL-3.0 licensed airline/airport data - -To include license-required sources: -```bash -# Interactive acceptance -skyview-data import openflights - -# Force acceptance (for automation) -skyview-data update openflights --force -``` - -## Monitoring - -### Check Update Status - -```bash -# View database status -skyview-data status - -# Example output: -# SkyView Database Status -# ====================== -# Database: /var/lib/skyview/skyview.db -# Size: 15.4 MB -# Modified: 2025-01-15T03:00:12Z -# -# šŸ“¦ Loaded Data Sources (2): -# ā€¢ OurAirports (Public Domain) -# ā€¢ FAA Registry (Public Domain) -# -# šŸ“Š Statistics: -# Aircraft History: 1,234 records -# Unique Aircraft: 567 -# Last 24h: 89 records -``` - -### Log Monitoring - -```bash -# View recent updates -tail -f /var/log/skyview-update.log - -# Check for errors -grep ERROR /var/log/skyview-update.log -``` - -## Troubleshooting - -### Common Issues - -#### Database Not Found -``` -ERROR: failed to create database: no write permission -``` -**Solution**: Ensure the skyview user has write access to `/var/lib/skyview/` - -#### Network Errors -``` -ERROR: failed to download data: connection timeout -``` -**Solution**: Check internet connectivity and firewall settings - -#### Lock File Issues -``` -ERROR: Another instance is already running -``` -**Solution**: Wait for current update to finish, or remove stale lock file - -### Manual Debugging - -```bash -# Verbose output -skyview-data -v update - -# Force update (skips locks) -skyview-data update --force - -# Reset database -skyview-data reset --force -``` - -## Advanced Configuration - -### Custom Update Script - -Create your own update script with custom logic: - -```bash -#!/bin/bash -# custom-update.sh - -# Only update if database is older than 7 days -if [ "$(find /var/lib/skyview/skyview.db -mtime +7)" ]; then - skyview-data update - systemctl reload skyview # Reload SkyView after update -fi -``` - -### Integration with SkyView Service - -```bash -# Reload SkyView after database updates -skyview-data update && systemctl reload skyview -``` - -### Backup Before Updates - -```bash -#!/bin/bash -# backup-and-update.sh - -DB_PATH="/var/lib/skyview/skyview.db" -BACKUP_DIR="/var/backups/skyview" - -# Create backup -mkdir -p "$BACKUP_DIR" -cp "$DB_PATH" "$BACKUP_DIR/skyview-$(date +%Y%m%d).db" - -# Keep only last 7 backups -find "$BACKUP_DIR" -name "skyview-*.db" -type f -mtime +7 -delete - -# Update database -skyview-data update -``` - -## Security Considerations - -### File Permissions - -```bash -# Secure database directory -sudo chown skyview:skyview /var/lib/skyview -sudo chmod 755 /var/lib/skyview -sudo chmod 644 /var/lib/skyview/skyview.db -``` - -### Network Security - -- Updates only download from trusted sources (GitHub, government sites) -- All downloads use HTTPS -- No sensitive data is transmitted -- Local processing only - -### Resource Limits - -```bash -# Limit resources in cron (optional) -0 3 * * * timeout 30m nice -n 10 skyview-data update -``` - -This setup ensures your SkyView installation maintains up-to-date aviation data automatically while respecting all license requirements and security best practices. \ No newline at end of file diff --git a/docs/DATABASE.md b/docs/DATABASE.md deleted file mode 100644 index 280d603..0000000 --- a/docs/DATABASE.md +++ /dev/null @@ -1,442 +0,0 @@ -# SkyView Database Architecture - -This document describes SkyView's SQLite database architecture, migration system, and integration approach for persistent data storage. - -## Overview - -SkyView uses a single SQLite database to store: -- **Historic aircraft data**: Position history, message counts, signal strength -- **Callsign lookup data**: Cached airline/airport information from external APIs -- **Embedded aviation data**: OpenFlights airline and airport databases - -## Database Design Principles - -### Embedded Architecture -- Single SQLite file for all persistent data -- No external database dependencies -- Self-contained deployment with embedded schemas -- Backward compatibility through versioned migrations - -### Performance Optimization -- Strategic indexing for time-series aircraft data -- Efficient lookups for callsign enhancement -- Configurable data retention policies -- Query optimization for real-time operations - -### Data Safety -- Atomic migration transactions -- Pre-migration backups for destructive changes -- Data loss warnings for schema changes -- Rollback capabilities where possible - -## Database Schema - -### Core Tables - -#### `schema_info` -Tracks database version and applied migrations: -```sql -CREATE TABLE schema_info ( - version INTEGER PRIMARY KEY, - applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - description TEXT, - checksum TEXT -); -``` - -#### `aircraft_history` -Stores time-series aircraft position and message data: -```sql -CREATE TABLE aircraft_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - icao_hex TEXT NOT NULL, - timestamp TIMESTAMP NOT NULL, - latitude REAL, - longitude REAL, - altitude INTEGER, - speed INTEGER, - track INTEGER, - vertical_rate INTEGER, - squawk TEXT, - callsign TEXT, - source_id TEXT, - signal_strength REAL, - message_count INTEGER DEFAULT 1 -); -``` - -**Indexes:** -- `idx_aircraft_history_icao_time`: Fast queries by aircraft and time range -- `idx_aircraft_history_timestamp`: Time-based cleanup and queries -- `idx_aircraft_history_callsign`: Callsign-based searches - -#### `airlines` -OpenFlights embedded airline database: -```sql -CREATE TABLE airlines ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - alias TEXT, - iata TEXT, - icao TEXT, - callsign TEXT, - country TEXT, - active BOOLEAN DEFAULT 1 -); -``` - -**Indexes:** -- `idx_airlines_icao`: ICAO code lookup (primary for callsign enhancement) -- `idx_airlines_iata`: IATA code lookup - -#### `airports` -OpenFlights embedded airport database: -```sql -CREATE TABLE airports ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - city TEXT, - country TEXT, - iata TEXT, - icao TEXT, - latitude REAL, - longitude REAL, - altitude INTEGER, - timezone_offset REAL, - dst_type TEXT, - timezone TEXT -); -``` - -**Indexes:** -- `idx_airports_icao`: ICAO code lookup -- `idx_airports_iata`: IATA code lookup - -#### `callsign_cache` -Caches external API lookups for callsign enhancement: -```sql -CREATE TABLE callsign_cache ( - callsign TEXT PRIMARY KEY, - airline_icao TEXT, - airline_name TEXT, - flight_number TEXT, - origin_iata TEXT, - destination_iata TEXT, - aircraft_type TEXT, - cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - expires_at TIMESTAMP, - source TEXT DEFAULT 'local' -); -``` - -**Indexes:** -- `idx_callsign_cache_expires`: Efficient cache cleanup - -## Database Location Strategy - -### Path Resolution Order -1. **Explicit configuration**: `database.path` in config file -2. **System service**: `/var/lib/skyview/skyview.db` -3. **User mode**: `~/.local/share/skyview/skyview.db` -4. **Fallback**: `./skyview.db` in current directory - -### Directory Permissions -- System: `root:root` with `755` permissions for `/var/lib/skyview/` -- User: User-owned directories with standard permissions -- Service: `skyview:skyview` user/group for system service - -## Migration System - -### Migration Structure -```go -type Migration struct { - Version int // Sequential version number - Description string // Human-readable description - Up string // SQL for applying migration - Down string // SQL for rollback (optional) - DataLoss bool // Warning flag for destructive changes -} -``` - -### Migration Process -1. **Version Check**: Compare current schema version with available migrations -2. **Backup**: Create automatic backup before destructive changes -3. **Transaction**: Wrap each migration in atomic transaction -4. **Validation**: Verify schema integrity after migration -5. **Logging**: Record successful migrations in `schema_info` - -### Data Loss Protection -- Migrations marked with `DataLoss: true` require explicit user consent -- Automatic backups created before destructive operations -- Warning messages displayed during upgrade process -- Rollback SQL provided where possible - -### Example Migration Sequence -```go -var migrations = []Migration{ - { - Version: 1, - Description: "Initial schema with aircraft history", - Up: createInitialSchema, - DataLoss: false, - }, - { - Version: 2, - Description: "Add OpenFlights airline and airport data", - Up: addAviationTables, - DataLoss: false, - }, - { - Version: 3, - Description: "Add callsign lookup cache", - Up: addCallsignCache, - DataLoss: false, - }, -} -``` - -## Configuration Integration - -### Database Configuration -```json -{ - "database": { - "path": "/var/lib/skyview/skyview.db", - "max_history_days": 7, - "backup_on_upgrade": true - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "external_apis": true, - "privacy_mode": false - } -} -``` - -### Configuration Fields - -#### `database` -- **`path`**: Database file location (empty = auto-resolve) -- **`max_history_days`**: Retention policy for aircraft history (0 = unlimited) -- **`backup_on_upgrade`**: Create backup before schema migrations - -#### `callsign` -- **`enabled`**: Enable callsign enhancement features -- **`cache_hours`**: TTL for cached external API results -- **`privacy_mode`**: Disable all external data requests -- **`sources`**: Independent control for each data source - -### Enhanced Configuration Example -```json -{ - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_embedded": { - "enabled": true, - "priority": 1, - "license": "AGPL-3.0" - }, - "faa_registry": { - "enabled": false, - "priority": 2, - "update_frequency": "weekly", - "license": "public_domain" - }, - "opensky_api": { - "enabled": false, - "priority": 3, - "timeout_seconds": 5, - "max_retries": 2, - "requires_consent": true, - "license_warning": "Commercial use requires OpenSky Network consent", - "user_accepts_terms": false - }, - "custom_database": { - "enabled": false, - "priority": 4, - "path": "", - "license": "user_verified" - } - }, - "fallback_chain": ["openflights_embedded", "faa_registry", "opensky_api", "custom_database"] - } -} -``` - -#### Individual Source Configuration Options -- **`enabled`**: Enable/disable this specific source -- **`priority`**: Processing order (lower numbers = higher priority) -- **`license`**: License type for compliance tracking -- **`requires_consent`**: Whether source requires explicit user consent -- **`user_accepts_terms`**: User acknowledgment of licensing terms -- **`timeout_seconds`**: Per-source timeout configuration -- **`max_retries`**: Per-source retry limits -- **`update_frequency`**: For downloadable sources (daily/weekly/monthly) - -## Debian Package Integration - -### Package Structure -``` -/var/lib/skyview/ # Database directory -/etc/skyview/config.json # Default configuration -/usr/bin/skyview # Main application -/usr/share/skyview/ # Embedded resources -``` - -### Installation Process -1. **`postinst`**: Create directories, user accounts, permissions -2. **First Run**: Database initialization and migration on startup -3. **Upgrades**: Automatic schema migration with backup -4. **Service**: Systemd integration with proper database access - -### Service User -- User: `skyview` -- Home: `/var/lib/skyview` -- Shell: `/bin/false` (service account) -- Database: Read/write access to `/var/lib/skyview/` - -## Data Retention and Cleanup - -### Automatic Cleanup -- **Aircraft History**: Configurable retention period (`max_history_days`) -- **Cache Expiration**: TTL-based cleanup of external API cache -- **Optimization**: Periodic VACUUM operations for storage efficiency - -### Manual Maintenance -```sql --- Clean old aircraft history (example: 7 days) -DELETE FROM aircraft_history -WHERE timestamp < datetime('now', '-7 days'); - --- Clean expired cache entries -DELETE FROM callsign_cache -WHERE expires_at < datetime('now'); - --- Optimize database storage -VACUUM; -``` - -## Performance Considerations - -### Query Optimization -- Time-range queries use `idx_aircraft_history_icao_time` -- Callsign lookups prioritize local cache over external APIs -- Bulk operations use transactions for consistency - -### Storage Efficiency -- Configurable history limits prevent unbounded growth -- Periodic VACUUM operations reclaim deleted space -- Compressed timestamps and efficient data types - -### Memory Usage -- WAL mode for concurrent read/write access -- Connection pooling for multiple goroutines -- Prepared statements for repeated queries - -## Privacy and Security - -### Privacy Mode -SkyView includes comprehensive privacy controls through the `privacy_mode` configuration option: - -```json -{ - "callsign": { - "enabled": true, - "privacy_mode": true, - "external_apis": false - } -} -``` - -#### Privacy Mode Features -- **No External Calls**: Completely disables all external API requests -- **Local-Only Lookups**: Uses only embedded OpenFlights database for callsign enhancement -- **No Data Transmission**: Aircraft data never leaves the local system -- **Compliance**: Suitable for sensitive environments requiring air-gapped operation - -#### Privacy Mode Behavior -| Feature | Privacy Mode ON | Privacy Mode OFF | -|---------|----------------|------------------| -| External API calls | āŒ Disabled | āœ… Configurable | -| OpenFlights lookup | āœ… Enabled | āœ… Enabled | -| Callsign caching | āœ… Local only | āœ… Full caching | -| Data transmission | āŒ None | āš ļø API calls only | - -#### Use Cases for Privacy Mode -- **Military installations**: No external data transmission allowed -- **Air-gapped networks**: No internet connectivity available -- **Corporate policies**: External API usage prohibited -- **Personal privacy**: User preference for local-only operation - -### Security Considerations - -#### File Permissions -- Database files readable only by skyview user/group -- Configuration files protected from unauthorized access -- Backup files inherit secure permissions - -#### Data Protection -- Local SQLite database with file-system level security -- No cloud storage or external database dependencies -- All aviation data processed and stored locally - -#### Network Security -- External API calls (when enabled) use HTTPS only -- No persistent connections to external services -- Optional certificate validation for API endpoints - -### Data Integrity -- Foreign key constraints where applicable -- Transaction isolation for concurrent operations -- Checksums for migration verification - -## Troubleshooting - -### Common Issues - -#### Database Locked -``` -Error: database is locked -``` -**Solution**: Stop SkyView service, check for stale lock files, restart - -#### Migration Failures -``` -Error: migration 3 failed: table already exists -``` -**Solution**: Check schema version, restore from backup, retry migration - -#### Permission Denied -``` -Error: unable to open database file -``` -**Solution**: Verify file permissions, check directory ownership, ensure disk space - -### Diagnostic Commands -```bash -# Check database integrity -sqlite3 /var/lib/skyview/skyview.db "PRAGMA integrity_check;" - -# View schema version -sqlite3 /var/lib/skyview/skyview.db "SELECT * FROM schema_info;" - -# Database statistics -sqlite3 /var/lib/skyview/skyview.db ".dbinfo" -``` - -## Future Enhancements - -### Planned Features -- **Compression**: Time-series compression for long-term storage -- **Partitioning**: Date-based partitioning for large datasets -- **Replication**: Read replica support for high-availability setups -- **Analytics**: Built-in reporting and statistics tables - -### Migration Path -- All enhancements will use versioned migrations -- Backward compatibility maintained for existing installations -- Data preservation prioritized over schema optimization \ No newline at end of file diff --git a/docs/MIGRATION_GUIDE.md b/docs/MIGRATION_GUIDE.md deleted file mode 100644 index c284e85..0000000 --- a/docs/MIGRATION_GUIDE.md +++ /dev/null @@ -1,340 +0,0 @@ -# SkyView Database Migration Guide - -This guide covers the transition from SkyView's in-memory data storage to persistent SQLite database storage, introduced in version 0.1.0. - -## Overview of Changes - -### What's New -- **Persistent Storage**: Aircraft position history survives restarts -- **Callsign Enhancement**: Enriched aircraft information with airline/airport data -- **Embedded Aviation Database**: OpenFlights airline and airport data included -- **Configurable Retention**: Control how long historical data is kept -- **Privacy Controls**: Comprehensive privacy mode for sensitive environments - -### What's Changed -- **Memory Usage**: Reduced memory footprint for aircraft tracking -- **Startup Time**: Slightly longer initial startup due to database initialization -- **Configuration**: New database and callsign configuration sections -- **File Structure**: New database file created in system or user directories - -## Pre-Migration Checklist - -### System Requirements -- **Disk Space**: Minimum 100MB available for database and backups -- **Permissions**: Write access to `/var/lib/skyview/` (system) or `~/.local/share/skyview/` (user) -- **SQLite**: No additional installation required (embedded in SkyView) - -### Current Data -āš ļø **Important**: In-memory aircraft data from previous versions cannot be preserved during migration. Historical tracking will start fresh after the upgrade. - -## Migration Process - -### Automatic Migration (Recommended) - -#### For Debian Package Users -```bash -# Update to new version -sudo apt update && sudo apt upgrade skyview-adsb - -# Service will automatically restart and initialize database -sudo systemctl status skyview -``` - -#### For Manual Installation Users -```bash -# Stop current SkyView instance -sudo systemctl stop skyview # or kill existing process - -# Backup current configuration -cp /etc/skyview/config.json /etc/skyview/config.json.backup - -# Start new version (database will be created automatically) -sudo systemctl start skyview -``` - -### Manual Database Setup - -If automatic initialization fails, you can manually initialize the database: - -```bash -# Create database directory -sudo mkdir -p /var/lib/skyview -sudo chown skyview:skyview /var/lib/skyview - -# Run SkyView with explicit database initialization -sudo -u skyview /usr/bin/skyview --init-database --config /etc/skyview/config.json -``` - -## Configuration Updates - -### New Configuration Sections - -Add these sections to your existing `config.json`: - -```json -{ - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "external_apis": true, - "privacy_mode": false - } -} -``` - -### Configuration Migration - -#### Default System Configuration -The new default configuration will be created at `/etc/skyview/config.json` if it doesn't exist: - -```json -{ - "server": { - "host": "", - "port": 8080 - }, - "sources": [ - { - "id": "local", - "name": "Local Receiver", - "host": "localhost", - "port": 30005, - "format": "beast", - "latitude": 0, - "longitude": 0, - "altitude": 0, - "enabled": true - } - ], - "settings": { - "history_limit": 500, - "stale_timeout": 60, - "update_rate": 1 - }, - "origin": { - "latitude": 51.4700, - "longitude": -0.4600, - "name": "Default Origin" - }, - "database": { - "path": "", - "max_history_days": 7, - "backup_on_upgrade": true - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "external_apis": true, - "privacy_mode": false - } -} -``` - -#### Preserving Custom Settings - -If you have customized your configuration, merge your existing settings with the new sections: - -```bash -# Backup original -cp /etc/skyview/config.json /etc/skyview/config.json.original - -# Edit configuration to add new sections -sudo nano /etc/skyview/config.json -``` - -## Privacy Configuration - -### Enabling Privacy Mode - -For sensitive environments that require no external data transmission: - -```json -{ - "callsign": { - "enabled": true, - "external_apis": false, - "privacy_mode": true, - "cache_hours": 168 - } -} -``` - -### Privacy Mode Features -- **No External API Calls**: Completely disables OpenSky Network and other external APIs -- **Local-Only Enhancement**: Uses embedded OpenFlights data for callsign lookup -- **Offline Operation**: Full functionality without internet connectivity -- **Compliance Ready**: Suitable for air-gapped or restricted networks - -## Post-Migration Verification - -### Database Verification -```bash -# Check database file exists and has correct permissions -ls -la /var/lib/skyview/skyview.db -# Should show: -rw-r--r-- 1 skyview skyview [size] [date] skyview.db - -# Verify database schema -sqlite3 /var/lib/skyview/skyview.db "SELECT version FROM schema_info ORDER BY version DESC LIMIT 1;" -# Should return current schema version number -``` - -### Service Health Check -```bash -# Check service status -sudo systemctl status skyview - -# Check logs for any errors -sudo journalctl -u skyview -f - -# Verify web interface accessibility -curl -I http://localhost:8080/ -# Should return: HTTP/1.1 200 OK -``` - -### Feature Testing -1. **Historical Data**: Verify aircraft positions persist after restart -2. **Callsign Enhancement**: Check that airline names appear for aircraft with callsigns -3. **Performance**: Monitor memory and CPU usage compared to previous version - -## Troubleshooting - -### Common Migration Issues - -#### Database Permission Errors -``` -Error: unable to open database file -``` - -**Solution**: -```bash -sudo chown -R skyview:skyview /var/lib/skyview/ -sudo chmod 755 /var/lib/skyview/ -sudo chmod 644 /var/lib/skyview/skyview.db -``` - -#### Migration Failures -``` -Error: migration failed at version X -``` - -**Solution**: -```bash -# Stop service -sudo systemctl stop skyview - -# Remove corrupted database -sudo rm /var/lib/skyview/skyview.db - -# Restart service (will recreate database) -sudo systemctl start skyview -``` - -#### Configuration Conflicts -``` -Error: unknown configuration field 'database' -``` - -**Solution**: Update configuration file with new sections, or reset to default configuration. - -### Rolling Back - -If you need to revert to the previous version: - -```bash -# Stop current service -sudo systemctl stop skyview - -# Install previous package version -sudo apt install skyview-adsb=[previous-version] - -# Remove database directory (optional) -sudo rm -rf /var/lib/skyview/ - -# Restore original configuration -sudo cp /etc/skyview/config.json.backup /etc/skyview/config.json - -# Start service -sudo systemctl start skyview -``` - -āš ļø **Note**: Rolling back will lose all historical aircraft data stored in the database. - -## Performance Impact - -### Expected Changes - -#### Memory Usage -- **Before**: ~50-100MB RAM for aircraft tracking -- **After**: ~30-60MB RAM (reduced due to database storage) - -#### Disk Usage -- **Database**: ~10-50MB depending on retention settings and traffic -- **Backups**: Additional ~10-50MB for migration backups - -#### Startup Time -- **Before**: 1-2 seconds -- **After**: 2-5 seconds (database initialization) - -### Optimization Recommendations - -#### For High-Traffic Environments -```json -{ - "database": { - "max_history_days": 3, - "backup_on_upgrade": false - }, - "settings": { - "history_limit": 100 - } -} -``` - -#### For Resource-Constrained Systems -```json -{ - "callsign": { - "enabled": false - }, - "database": { - "max_history_days": 1 - } -} -``` - -## Benefits After Migration - -### Enhanced Features -- **Persistent History**: Aircraft tracks survive system restarts -- **Rich Callsign Data**: Airline names, routes, and aircraft types -- **Better Analytics**: Historical data enables trend analysis -- **Improved Performance**: Reduced memory usage for long-running instances - -### Operational Improvements -- **Service Reliability**: Database recovery after crashes -- **Maintenance Windows**: Graceful restart without data loss -- **Monitoring**: Historical data for performance analysis -- **Compliance**: Privacy controls for regulatory requirements - -## Support - -### Getting Help -- **Documentation**: Check `/usr/share/doc/skyview-adsb/` for additional guides -- **Logs**: Service logs available via `journalctl -u skyview` -- **Configuration**: Example configs in `/usr/share/skyview/examples/` -- **Community**: Report issues at project repository - -### Reporting Issues -When reporting migration issues, include: -- SkyView version (before and after) -- Operating system and version -- Configuration file content -- Error messages from logs -- Database file permissions (`ls -la /var/lib/skyview/`) - -This migration enables SkyView's evolution toward more sophisticated aircraft tracking while maintaining the simplicity and reliability of the existing system. \ No newline at end of file diff --git a/go.mod b/go.mod index 24b62db..fed2562 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,3 @@ require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 ) - -require github.com/mattn/go-sqlite3 v1.14.32 // indirect diff --git a/go.sum b/go.sum index f14e497..7ed87b7 100644 --- a/go.sum +++ b/go.sum @@ -2,5 +2,3 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= -github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= diff --git a/internal/database/api_client.go b/internal/database/api_client.go deleted file mode 100644 index f7b7614..0000000 --- a/internal/database/api_client.go +++ /dev/null @@ -1,389 +0,0 @@ -package database - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "sync" - "time" -) - -type ExternalAPIClient struct { - httpClient *http.Client - mutex sync.RWMutex - - // Configuration - timeout time.Duration - maxRetries int - userAgent string - - // Rate limiting - lastRequest time.Time - minInterval time.Duration -} - -type APIClientConfig struct { - Timeout time.Duration - MaxRetries int - UserAgent string - MinInterval time.Duration // Minimum interval between requests -} - -type OpenSkyFlightInfo struct { - ICAO string `json:"icao"` - Callsign string `json:"callsign"` - Origin string `json:"origin"` - Destination string `json:"destination"` - FirstSeen time.Time `json:"first_seen"` - LastSeen time.Time `json:"last_seen"` - AircraftType string `json:"aircraft_type"` - Registration string `json:"registration"` - FlightNumber string `json:"flight_number"` - Airline string `json:"airline"` -} - -type APIError struct { - Operation string - StatusCode int - Message string - Retryable bool - RetryAfter time.Duration -} - -func (e *APIError) Error() string { - return fmt.Sprintf("API error in %s: %s (status: %d, retryable: %v)", - e.Operation, e.Message, e.StatusCode, e.Retryable) -} - -func NewExternalAPIClient(config APIClientConfig) *ExternalAPIClient { - if config.Timeout == 0 { - config.Timeout = 10 * time.Second - } - if config.MaxRetries == 0 { - config.MaxRetries = 3 - } - if config.UserAgent == "" { - config.UserAgent = "SkyView-ADSB/1.0 (https://github.com/user/skyview)" - } - if config.MinInterval == 0 { - config.MinInterval = 1 * time.Second // Default rate limit - } - - return &ExternalAPIClient{ - httpClient: &http.Client{ - Timeout: config.Timeout, - }, - timeout: config.Timeout, - maxRetries: config.MaxRetries, - userAgent: config.UserAgent, - minInterval: config.MinInterval, - } -} - -func (c *ExternalAPIClient) enforceRateLimit() { - c.mutex.Lock() - defer c.mutex.Unlock() - - elapsed := time.Since(c.lastRequest) - if elapsed < c.minInterval { - time.Sleep(c.minInterval - elapsed) - } - c.lastRequest = time.Now() -} - -func (c *ExternalAPIClient) makeRequest(ctx context.Context, url string) (*http.Response, error) { - c.enforceRateLimit() - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", c.userAgent) - req.Header.Set("Accept", "application/json") - - var resp *http.Response - var lastErr error - - for attempt := 0; attempt <= c.maxRetries; attempt++ { - if attempt > 0 { - // Exponential backoff - backoff := time.Duration(1<= 500 || resp.StatusCode == 429 { - resp.Body.Close() - - // Handle rate limiting - if resp.StatusCode == 429 { - retryAfter := parseRetryAfter(resp.Header.Get("Retry-After")) - if retryAfter > 0 { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(retryAfter): - } - } - } - continue - } - - // Success or non-retryable error - break - } - - if lastErr != nil { - return nil, lastErr - } - - return resp, nil -} - -func (c *ExternalAPIClient) GetFlightInfoFromOpenSky(ctx context.Context, icao string) (*OpenSkyFlightInfo, error) { - if icao == "" { - return nil, fmt.Errorf("empty ICAO code") - } - - // OpenSky Network API endpoint for flight information - apiURL := fmt.Sprintf("https://opensky-network.org/api/flights/aircraft?icao24=%s&begin=%d&end=%d", - icao, - time.Now().Add(-24*time.Hour).Unix(), - time.Now().Unix(), - ) - - resp, err := c.makeRequest(ctx, apiURL) - if err != nil { - return nil, &APIError{ - Operation: "opensky_flight_info", - Message: err.Error(), - Retryable: true, - } - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, &APIError{ - Operation: "opensky_flight_info", - StatusCode: resp.StatusCode, - Message: string(body), - Retryable: resp.StatusCode >= 500 || resp.StatusCode == 429, - } - } - - var flights [][]interface{} - decoder := json.NewDecoder(resp.Body) - if err := decoder.Decode(&flights); err != nil { - return nil, &APIError{ - Operation: "opensky_parse_response", - Message: err.Error(), - Retryable: false, - } - } - - if len(flights) == 0 { - return nil, nil // No flight information available - } - - // Parse the most recent flight - flight := flights[0] - if len(flight) < 10 { - return nil, &APIError{ - Operation: "opensky_invalid_response", - Message: "invalid flight data format", - Retryable: false, - } - } - - info := &OpenSkyFlightInfo{ - ICAO: icao, - } - - // Parse fields based on OpenSky API documentation - if callsign, ok := flight[1].(string); ok { - info.Callsign = callsign - } - if firstSeen, ok := flight[2].(float64); ok { - info.FirstSeen = time.Unix(int64(firstSeen), 0) - } - if lastSeen, ok := flight[3].(float64); ok { - info.LastSeen = time.Unix(int64(lastSeen), 0) - } - if origin, ok := flight[4].(string); ok { - info.Origin = origin - } - if destination, ok := flight[5].(string); ok { - info.Destination = destination - } - - return info, nil -} - -func (c *ExternalAPIClient) GetAircraftInfoFromOpenSky(ctx context.Context, icao string) (map[string]interface{}, error) { - if icao == "" { - return nil, fmt.Errorf("empty ICAO code") - } - - // OpenSky Network metadata API - apiURL := fmt.Sprintf("https://opensky-network.org/api/metadata/aircraft/icao/%s", icao) - - resp, err := c.makeRequest(ctx, apiURL) - if err != nil { - return nil, &APIError{ - Operation: "opensky_aircraft_info", - Message: err.Error(), - Retryable: true, - } - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusNotFound { - return nil, nil // Aircraft not found - } - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, &APIError{ - Operation: "opensky_aircraft_info", - StatusCode: resp.StatusCode, - Message: string(body), - Retryable: resp.StatusCode >= 500 || resp.StatusCode == 429, - } - } - - var aircraft map[string]interface{} - decoder := json.NewDecoder(resp.Body) - if err := decoder.Decode(&aircraft); err != nil { - return nil, &APIError{ - Operation: "opensky_parse_aircraft", - Message: err.Error(), - Retryable: false, - } - } - - return aircraft, nil -} - -func (c *ExternalAPIClient) EnhanceCallsignWithExternalData(ctx context.Context, callsign, icao string) (map[string]interface{}, error) { - enhancement := make(map[string]interface{}) - enhancement["callsign"] = callsign - enhancement["icao"] = icao - enhancement["enhanced"] = false - - // Try to get flight information from OpenSky - if flightInfo, err := c.GetFlightInfoFromOpenSky(ctx, icao); err == nil && flightInfo != nil { - enhancement["flight_info"] = map[string]interface{}{ - "origin": flightInfo.Origin, - "destination": flightInfo.Destination, - "first_seen": flightInfo.FirstSeen, - "last_seen": flightInfo.LastSeen, - "flight_number": flightInfo.FlightNumber, - "airline": flightInfo.Airline, - } - enhancement["enhanced"] = true - } - - // Try to get aircraft metadata - if aircraftInfo, err := c.GetAircraftInfoFromOpenSky(ctx, icao); err == nil && aircraftInfo != nil { - enhancement["aircraft_info"] = aircraftInfo - enhancement["enhanced"] = true - } - - return enhancement, nil -} - -func (c *ExternalAPIClient) BatchEnhanceCallsigns(ctx context.Context, callsigns map[string]string) (map[string]map[string]interface{}, error) { - results := make(map[string]map[string]interface{}) - - for callsign, icao := range callsigns { - select { - case <-ctx.Done(): - return results, ctx.Err() - default: - } - - enhanced, err := c.EnhanceCallsignWithExternalData(ctx, callsign, icao) - if err != nil { - // Log error but continue with other callsigns - fmt.Printf("Warning: failed to enhance callsign %s (ICAO: %s): %v\n", callsign, icao, err) - continue - } - - results[callsign] = enhanced - } - - return results, nil -} - -func (c *ExternalAPIClient) TestConnection(ctx context.Context) error { - // Test with a simple API call - testURL := "https://opensky-network.org/api/states?time=0&lamin=0&lomin=0&lamax=1&lomax=1" - - resp, err := c.makeRequest(ctx, testURL) - if err != nil { - return fmt.Errorf("connection test failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("connection test returned status %d", resp.StatusCode) - } - - return nil -} - -func parseRetryAfter(header string) time.Duration { - if header == "" { - return 0 - } - - // Try parsing as seconds - if seconds, err := time.ParseDuration(header + "s"); err == nil { - return seconds - } - - // Try parsing as HTTP date - if t, err := http.ParseTime(header); err == nil { - return time.Until(t) - } - - return 0 -} - -// HealthCheck provides information about the client's health -func (c *ExternalAPIClient) HealthCheck(ctx context.Context) map[string]interface{} { - health := make(map[string]interface{}) - - // Test connection - if err := c.TestConnection(ctx); err != nil { - health["status"] = "unhealthy" - health["error"] = err.Error() - } else { - health["status"] = "healthy" - } - - // Add configuration info - health["timeout"] = c.timeout.String() - health["max_retries"] = c.maxRetries - health["min_interval"] = c.minInterval.String() - health["user_agent"] = c.userAgent - - c.mutex.RLock() - health["last_request"] = c.lastRequest - c.mutex.RUnlock() - - return health -} \ No newline at end of file diff --git a/internal/database/database.go b/internal/database/database.go deleted file mode 100644 index 8a44418..0000000 --- a/internal/database/database.go +++ /dev/null @@ -1,256 +0,0 @@ -// Package database provides persistent storage for aircraft data and callsign enhancement -// using SQLite with versioned schema migrations and comprehensive error handling. -// -// The database system supports: -// - Aircraft position history with configurable retention -// - Embedded OpenFlights airline and airport data -// - External API result caching with TTL -// - Schema migrations with rollback support -// - Privacy mode for complete offline operation -package database - -import ( - "database/sql" - "fmt" - "time" - - _ "github.com/mattn/go-sqlite3" // SQLite driver -) - -// Database represents the main database connection and operations -type Database struct { - conn *sql.DB - config *Config - migrator *Migrator - callsign *CallsignManager - history *HistoryManager -} - -// Config holds database configuration options -type Config struct { - // Database file path (auto-resolved if empty) - Path string `json:"path"` - - // Data retention settings - MaxHistoryDays int `json:"max_history_days"` // 0 = unlimited - BackupOnUpgrade bool `json:"backup_on_upgrade"` - - // Connection settings - MaxOpenConns int `json:"max_open_conns"` // Default: 10 - MaxIdleConns int `json:"max_idle_conns"` // Default: 5 - ConnMaxLifetime time.Duration `json:"conn_max_lifetime"` // Default: 1 hour - - // Maintenance settings - VacuumInterval time.Duration `json:"vacuum_interval"` // Default: 24 hours - CleanupInterval time.Duration `json:"cleanup_interval"` // Default: 1 hour -} - -// AircraftHistoryRecord represents a stored aircraft position update -type AircraftHistoryRecord struct { - ID int64 `json:"id"` - ICAO string `json:"icao"` - Timestamp time.Time `json:"timestamp"` - Latitude *float64 `json:"latitude,omitempty"` - Longitude *float64 `json:"longitude,omitempty"` - Altitude *int `json:"altitude,omitempty"` - Speed *int `json:"speed,omitempty"` - Track *int `json:"track,omitempty"` - VerticalRate *int `json:"vertical_rate,omitempty"` - Squawk *string `json:"squawk,omitempty"` - Callsign *string `json:"callsign,omitempty"` - SourceID string `json:"source_id"` - SignalStrength *float64 `json:"signal_strength,omitempty"` -} - -// CallsignInfo represents enriched callsign information -type CallsignInfo struct { - OriginalCallsign string `json:"original_callsign"` - AirlineCode string `json:"airline_code"` - FlightNumber string `json:"flight_number"` - AirlineName string `json:"airline_name"` - AirlineCountry string `json:"airline_country"` - DisplayName string `json:"display_name"` - IsValid bool `json:"is_valid"` - LastUpdated time.Time `json:"last_updated"` -} - -// AirlineRecord represents embedded airline data from OpenFlights -type AirlineRecord struct { - ID int `json:"id"` - Name string `json:"name"` - Alias string `json:"alias"` - IATACode string `json:"iata_code"` - ICAOCode string `json:"icao_code"` - Callsign string `json:"callsign"` - Country string `json:"country"` - Active bool `json:"active"` -} - -// AirportRecord represents embedded airport data from OpenFlights -type AirportRecord struct { - ID int `json:"id"` - Name string `json:"name"` - City string `json:"city"` - Country string `json:"country"` - IATA string `json:"iata"` - ICAO string `json:"icao"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Altitude int `json:"altitude"` - TimezoneOffset float64 `json:"timezone_offset"` - DST string `json:"dst"` - Timezone string `json:"timezone"` -} - -// DatabaseError represents database operation errors -type DatabaseError struct { - Operation string - Err error - Query string - Retryable bool -} - -func (e *DatabaseError) Error() string { - if e.Query != "" { - return fmt.Sprintf("database %s error: %v (query: %s)", e.Operation, e.Err, e.Query) - } - return fmt.Sprintf("database %s error: %v", e.Operation, e.Err) -} - -func (e *DatabaseError) Unwrap() error { - return e.Err -} - -// NewDatabase creates a new database connection with the given configuration -func NewDatabase(config *Config) (*Database, error) { - if config == nil { - config = DefaultConfig() - } - - // Resolve database path - dbPath, err := ResolveDatabasePath(config.Path) - if err != nil { - return nil, &DatabaseError{ - Operation: "path_resolution", - Err: err, - Retryable: false, - } - } - config.Path = dbPath - - // Open database connection - conn, err := sql.Open("sqlite3", buildConnectionString(dbPath)) - if err != nil { - return nil, &DatabaseError{ - Operation: "connect", - Err: err, - Retryable: true, - } - } - - // Configure connection pool - conn.SetMaxOpenConns(config.MaxOpenConns) - conn.SetMaxIdleConns(config.MaxIdleConns) - conn.SetConnMaxLifetime(config.ConnMaxLifetime) - - // Test connection - if err := conn.Ping(); err != nil { - conn.Close() - return nil, &DatabaseError{ - Operation: "ping", - Err: err, - Retryable: true, - } - } - - db := &Database{ - conn: conn, - config: config, - } - - // Initialize components - db.migrator = NewMigrator(conn) - db.callsign = NewCallsignManager(conn) - db.history = NewHistoryManager(conn, config.MaxHistoryDays) - - return db, nil -} - -// Initialize runs database migrations and sets up embedded data -func (db *Database) Initialize() error { - // Run schema migrations - if err := db.migrator.MigrateToLatest(); err != nil { - return &DatabaseError{ - Operation: "migration", - Err: err, - Retryable: false, - } - } - - // Load embedded OpenFlights data if not already loaded - if err := db.callsign.LoadEmbeddedData(); err != nil { - return &DatabaseError{ - Operation: "load_embedded_data", - Err: err, - Retryable: false, - } - } - - return nil -} - -// GetConfig returns the database configuration -func (db *Database) GetConfig() *Config { - return db.config -} - -// GetConnection returns the underlying database connection -func (db *Database) GetConnection() *sql.DB { - return db.conn -} - -// GetHistoryManager returns the history manager -func (db *Database) GetHistoryManager() *HistoryManager { - return db.history -} - -// GetCallsignManager returns the callsign manager -func (db *Database) GetCallsignManager() *CallsignManager { - return db.callsign -} - -// Close closes the database connection and stops background tasks -func (db *Database) Close() error { - if db.conn != nil { - return db.conn.Close() - } - return nil -} - -// Health returns the database health status -func (db *Database) Health() error { - if db.conn == nil { - return fmt.Errorf("database connection not initialized") - } - return db.conn.Ping() -} - - -// DefaultConfig returns the default database configuration -func DefaultConfig() *Config { - return &Config{ - Path: "", // Auto-resolved - MaxHistoryDays: 7, - BackupOnUpgrade: true, - MaxOpenConns: 10, - MaxIdleConns: 5, - ConnMaxLifetime: time.Hour, - VacuumInterval: 24 * time.Hour, - CleanupInterval: time.Hour, - } -} - -// buildConnectionString creates SQLite connection string with optimizations -func buildConnectionString(path string) string { - return fmt.Sprintf("%s?_journal_mode=WAL&_synchronous=NORMAL&_cache_size=-64000&_temp_store=MEMORY&_foreign_keys=ON", path) -} \ No newline at end of file diff --git a/internal/database/loader.go b/internal/database/loader.go deleted file mode 100644 index 04b4653..0000000 --- a/internal/database/loader.go +++ /dev/null @@ -1,526 +0,0 @@ -// Package database - Data loader for external sources -// -// This module handles loading aviation data from external sources at runtime, -// maintaining license compliance by not embedding any AGPL or restricted data -// in the SkyView binary. -package database - -import ( - "crypto/tls" - "database/sql" - "encoding/csv" - "fmt" - "io" - "net/http" - "os" - "strconv" - "strings" - "time" -) - -// DataLoader handles loading external data sources into the database -type DataLoader struct { - conn *sql.DB - client *http.Client -} - -// DataSource represents an external aviation data source -type DataSource struct { - Name string `json:"name"` - License string `json:"license"` - URL string `json:"url"` - RequiresConsent bool `json:"requires_consent"` - UserAcceptedLicense bool `json:"user_accepted_license"` - Format string `json:"format"` // "openflights", "ourairports", "csv" - Version string `json:"version"` -} - -// LoadResult contains the results of a data loading operation -type LoadResult struct { - Source string `json:"source"` - RecordsTotal int `json:"records_total"` - RecordsNew int `json:"records_new"` - RecordsError int `json:"records_error"` - Duration time.Duration `json:"duration"` - Errors []string `json:"errors,omitempty"` -} - -// NewDataLoader creates a new data loader with HTTP client -func NewDataLoader(conn *sql.DB) *DataLoader { - // Check for insecure TLS environment variable - insecureTLS := os.Getenv("SKYVIEW_INSECURE_TLS") == "1" - - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 90 * time.Second, - DisableCompression: false, - } - - // Allow insecure certificates if requested - if insecureTLS { - transport.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } - - return &DataLoader{ - conn: conn, - client: &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - }, - } -} - -// GetAvailableDataSources returns all supported data sources with license info -func GetAvailableDataSources() []DataSource { - return []DataSource{ - { - Name: "OpenFlights Airlines", - License: "AGPL-3.0", - URL: "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat", - RequiresConsent: true, - Format: "openflights", - Version: "latest", - }, - { - Name: "OpenFlights Airports", - License: "AGPL-3.0", - URL: "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat", - RequiresConsent: true, - Format: "openflights", - Version: "latest", - }, - { - Name: "OurAirports", - License: "Public Domain", - URL: "https://raw.githubusercontent.com/davidmegginson/ourairports-data/main/airports.csv", - RequiresConsent: false, - Format: "ourairports", - Version: "latest", - }, - } -} - -// LoadDataSource downloads and imports data from an external source -func (dl *DataLoader) LoadDataSource(source DataSource) (*LoadResult, error) { - result := &LoadResult{ - Source: source.Name, - } - startTime := time.Now() - defer func() { - result.Duration = time.Since(startTime) - }() - - // Check license acceptance if required - if source.RequiresConsent && !source.UserAcceptedLicense { - return nil, fmt.Errorf("user has not accepted license for source: %s (%s)", source.Name, source.License) - } - - // Download data - resp, err := dl.client.Get(source.URL) - if err != nil { - return nil, fmt.Errorf("failed to download data from %s: %v", source.URL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP error downloading data: %s", resp.Status) - } - - // Parse and load data based on format - switch source.Format { - case "openflights": - if strings.Contains(source.Name, "Airlines") { - return dl.loadOpenFlightsAirlines(resp.Body, source, result) - } else if strings.Contains(source.Name, "Airports") { - return dl.loadOpenFlightsAirports(resp.Body, source, result) - } - return nil, fmt.Errorf("unknown OpenFlights data type: %s", source.Name) - - case "ourairports": - return dl.loadOurAirports(resp.Body, source, result) - - default: - return nil, fmt.Errorf("unsupported data format: %s", source.Format) - } -} - -// loadOpenFlightsAirlines loads airline data in OpenFlights format -func (dl *DataLoader) loadOpenFlightsAirlines(reader io.Reader, source DataSource, result *LoadResult) (*LoadResult, error) { - tx, err := dl.conn.Begin() - if err != nil { - return nil, fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Record data source - if err := dl.recordDataSource(tx, source); err != nil { - return nil, err - } - - // Clear existing data from this source - _, err = tx.Exec(`DELETE FROM airlines WHERE data_source = ?`, source.Name) - if err != nil { - return nil, fmt.Errorf("failed to clear existing airline data: %v", err) - } - - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 // Variable number of fields - - insertStmt, err := tx.Prepare(` - INSERT INTO airlines (id, name, alias, iata, icao, callsign, country, active, data_source) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - return nil, fmt.Errorf("failed to prepare insert statement: %v", err) - } - defer insertStmt.Close() - - for { - record, err := csvReader.Read() - if err == io.EOF { - break - } - if err != nil { - result.RecordsError++ - result.Errors = append(result.Errors, fmt.Sprintf("CSV parse error: %v", err)) - continue - } - - if len(record) < 7 { - result.RecordsError++ - result.Errors = append(result.Errors, "insufficient fields in record") - continue - } - - result.RecordsTotal++ - - // Parse OpenFlights airline format: - // ID, Name, Alias, IATA, ICAO, Callsign, Country, Active - id, _ := strconv.Atoi(record[0]) - name := strings.Trim(record[1], `"`) - alias := strings.Trim(record[2], `"`) - iata := strings.Trim(record[3], `"`) - icao := strings.Trim(record[4], `"`) - callsign := strings.Trim(record[5], `"`) - country := strings.Trim(record[6], `"`) - active := len(record) > 7 && strings.Trim(record[7], `"`) == "Y" - - // Convert \N to empty strings - if alias == "\\N" { alias = "" } - if iata == "\\N" { iata = "" } - if icao == "\\N" { icao = "" } - if callsign == "\\N" { callsign = "" } - - _, err = insertStmt.Exec(id, name, alias, iata, icao, callsign, country, active, source.Name) - if err != nil { - result.RecordsError++ - result.Errors = append(result.Errors, fmt.Sprintf("insert error for airline %s: %v", name, err)) - continue - } - - result.RecordsNew++ - } - - // Update record count - _, err = tx.Exec(`UPDATE data_sources SET record_count = ? WHERE name = ?`, result.RecordsNew, source.Name) - if err != nil { - return nil, fmt.Errorf("failed to update record count: %v", err) - } - - return result, tx.Commit() -} - -// loadOpenFlightsAirports loads airport data in OpenFlights format -func (dl *DataLoader) loadOpenFlightsAirports(reader io.Reader, source DataSource, result *LoadResult) (*LoadResult, error) { - tx, err := dl.conn.Begin() - if err != nil { - return nil, fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Record data source - if err := dl.recordDataSource(tx, source); err != nil { - return nil, err - } - - // Clear existing data from this source - _, err = tx.Exec(`DELETE FROM airports WHERE data_source = ?`, source.Name) - if err != nil { - return nil, fmt.Errorf("failed to clear existing airport data: %v", err) - } - - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 - - insertStmt, err := tx.Prepare(` - INSERT INTO airports (id, name, city, country, iata, icao, latitude, longitude, - altitude, timezone_offset, dst_type, timezone, data_source) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - return nil, fmt.Errorf("failed to prepare insert statement: %v", err) - } - defer insertStmt.Close() - - for { - record, err := csvReader.Read() - if err == io.EOF { - break - } - if err != nil { - result.RecordsError++ - result.Errors = append(result.Errors, fmt.Sprintf("CSV parse error: %v", err)) - continue - } - - if len(record) < 12 { - result.RecordsError++ - result.Errors = append(result.Errors, "insufficient fields in airport record") - continue - } - - result.RecordsTotal++ - - // Parse OpenFlights airport format - id, _ := strconv.Atoi(record[0]) - name := strings.Trim(record[1], `"`) - city := strings.Trim(record[2], `"`) - country := strings.Trim(record[3], `"`) - iata := strings.Trim(record[4], `"`) - icao := strings.Trim(record[5], `"`) - lat, _ := strconv.ParseFloat(record[6], 64) - lon, _ := strconv.ParseFloat(record[7], 64) - alt, _ := strconv.Atoi(record[8]) - tzOffset, _ := strconv.ParseFloat(record[9], 64) - dst := strings.Trim(record[10], `"`) - timezone := strings.Trim(record[11], `"`) - - // Convert \N to empty strings - if iata == "\\N" { iata = "" } - if icao == "\\N" { icao = "" } - if dst == "\\N" { dst = "" } - if timezone == "\\N" { timezone = "" } - - _, err = insertStmt.Exec(id, name, city, country, iata, icao, lat, lon, alt, tzOffset, dst, timezone, source.Name) - if err != nil { - result.RecordsError++ - result.Errors = append(result.Errors, fmt.Sprintf("insert error for airport %s: %v", name, err)) - continue - } - - result.RecordsNew++ - } - - // Update record count - _, err = tx.Exec(`UPDATE data_sources SET record_count = ? WHERE name = ?`, result.RecordsNew, source.Name) - if err != nil { - return nil, fmt.Errorf("failed to update record count: %v", err) - } - - return result, tx.Commit() -} - -// loadOurAirports loads airport data in OurAirports CSV format -func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, result *LoadResult) (*LoadResult, error) { - // Start database transaction - tx, err := dl.conn.Begin() - if err != nil { - return nil, fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - csvReader := csv.NewReader(reader) - - // Read header row - headers, err := csvReader.Read() - if err != nil { - result.RecordsError = 1 - result.Errors = []string{fmt.Sprintf("Failed to read CSV header: %v", err)} - return result, err - } - - // Create header index map for easier field access - headerIndex := make(map[string]int) - for i, header := range headers { - headerIndex[strings.TrimSpace(header)] = i - } - - // Prepare statement for airports - stmt, err := tx.Prepare(` - INSERT OR REPLACE INTO airports ( - source_id, name, ident, type, icao_code, iata_code, - latitude, longitude, elevation_ft, country_code, - municipality, continent, scheduled_service, - home_link, wikipedia_link, keywords, data_source - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - result.RecordsError = 1 - result.Errors = []string{fmt.Sprintf("Failed to prepare statement: %v", err)} - return result, err - } - defer stmt.Close() - - // Process each row - for { - record, err := csvReader.Read() - if err == io.EOF { - break - } - if err != nil { - result.RecordsError++ - result.Errors = append(result.Errors, fmt.Sprintf("CSV read error: %v", err)) - continue - } - - // Skip rows with insufficient fields - if len(record) < len(headerIndex) { - result.RecordsError++ - continue - } - - // Extract fields using header index - sourceID := getFieldByHeader(record, headerIndex, "id") - ident := getFieldByHeader(record, headerIndex, "ident") - name := getFieldByHeader(record, headerIndex, "name") - icaoCode := getFieldByHeader(record, headerIndex, "icao_code") - iataCode := getFieldByHeader(record, headerIndex, "iata_code") - airportType := getFieldByHeader(record, headerIndex, "type") - countryCode := getFieldByHeader(record, headerIndex, "iso_country") - municipality := getFieldByHeader(record, headerIndex, "municipality") - continent := getFieldByHeader(record, headerIndex, "continent") - homeLink := getFieldByHeader(record, headerIndex, "home_link") - wikipediaLink := getFieldByHeader(record, headerIndex, "wikipedia_link") - keywords := getFieldByHeader(record, headerIndex, "keywords") - - // Parse coordinates - var latitude, longitude float64 - if latStr := getFieldByHeader(record, headerIndex, "latitude_deg"); latStr != "" { - if lat, err := strconv.ParseFloat(latStr, 64); err == nil { - latitude = lat - } - } - if lngStr := getFieldByHeader(record, headerIndex, "longitude_deg"); lngStr != "" { - if lng, err := strconv.ParseFloat(lngStr, 64); err == nil { - longitude = lng - } - } - - // Parse elevation - var elevation int - if elevStr := getFieldByHeader(record, headerIndex, "elevation_ft"); elevStr != "" { - if elev, err := strconv.Atoi(elevStr); err == nil { - elevation = elev - } - } - - // Parse scheduled service - scheduledService := getFieldByHeader(record, headerIndex, "scheduled_service") == "yes" - - // Insert airport record - _, err = stmt.Exec( - sourceID, name, ident, airportType, icaoCode, iataCode, - latitude, longitude, elevation, countryCode, municipality, continent, - scheduledService, homeLink, wikipediaLink, keywords, source.Name, - ) - if err != nil { - result.RecordsError++ - result.Errors = append(result.Errors, fmt.Sprintf("Insert error for %s: %v", ident, err)) - } else { - result.RecordsNew++ - } - } - - // Update data source tracking - _, err = tx.Exec(` - INSERT OR REPLACE INTO data_sources (name, license, url, imported_at, record_count, user_accepted_license) - VALUES (?, ?, ?, CURRENT_TIMESTAMP, ?, ?) - `, source.Name, source.License, source.URL, result.RecordsNew, source.UserAcceptedLicense) - if err != nil { - return result, fmt.Errorf("failed to update data source tracking: %v", err) - } - - return result, tx.Commit() -} - -// getFieldByHeader safely gets a field value by header name -func getFieldByHeader(record []string, headerIndex map[string]int, fieldName string) string { - if idx, exists := headerIndex[fieldName]; exists && idx < len(record) { - return strings.TrimSpace(record[idx]) - } - return "" -} - -// GetLoadedDataSources returns all data sources that have been imported -func (dl *DataLoader) GetLoadedDataSources() ([]DataSource, error) { - query := ` - SELECT name, license, url, COALESCE(version, 'latest'), user_accepted_license - FROM data_sources - ORDER BY name - ` - - rows, err := dl.conn.Query(query) - if err != nil { - return nil, err - } - defer rows.Close() - - var sources []DataSource - for rows.Next() { - var source DataSource - err := rows.Scan( - &source.Name, - &source.License, - &source.URL, - &source.Version, - &source.UserAcceptedLicense, - ) - if err != nil { - return nil, err - } - sources = append(sources, source) - } - - return sources, rows.Err() -} - -// recordDataSource records information about the data source being imported -func (dl *DataLoader) recordDataSource(tx *sql.Tx, source DataSource) error { - _, err := tx.Exec(` - INSERT OR REPLACE INTO data_sources - (name, license, url, version, user_accepted_license) - VALUES (?, ?, ?, ?, ?) - `, source.Name, source.License, source.URL, source.Version, source.UserAcceptedLicense) - - return err -} - - -// ClearDataSource removes all data from a specific source -func (dl *DataLoader) ClearDataSource(sourceName string) error { - tx, err := dl.conn.Begin() - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Clear from all tables - _, err = tx.Exec(`DELETE FROM airlines WHERE data_source = ?`, sourceName) - if err != nil { - return fmt.Errorf("failed to clear airlines: %v", err) - } - - _, err = tx.Exec(`DELETE FROM airports WHERE data_source = ?`, sourceName) - if err != nil { - return fmt.Errorf("failed to clear airports: %v", err) - } - - _, err = tx.Exec(`DELETE FROM data_sources WHERE name = ?`, sourceName) - if err != nil { - return fmt.Errorf("failed to clear data source record: %v", err) - } - - return tx.Commit() -} \ No newline at end of file diff --git a/internal/database/manager_callsign.go b/internal/database/manager_callsign.go deleted file mode 100644 index 2f563c7..0000000 --- a/internal/database/manager_callsign.go +++ /dev/null @@ -1,362 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "regexp" - "strings" - "sync" - "time" -) - -type CallsignManager struct { - db *sql.DB - mutex sync.RWMutex - - // Compiled regex patterns for callsign parsing - airlinePattern *regexp.Regexp - flightPattern *regexp.Regexp -} - -type CallsignParseResult struct { - OriginalCallsign string - AirlineCode string - FlightNumber string - IsValid bool - ParsedTime time.Time -} - -func NewCallsignManager(db *sql.DB) *CallsignManager { - return &CallsignManager{ - db: db, - // Match airline code (2-3 letters) followed by flight number (1-4 digits, optional letter) - airlinePattern: regexp.MustCompile(`^([A-Z]{2,3})([0-9]{1,4}[A-Z]?)$`), - // More flexible pattern for general flight identification - flightPattern: regexp.MustCompile(`^([A-Z0-9]+)([0-9]+[A-Z]?)$`), - } -} - -func (cm *CallsignManager) ParseCallsign(callsign string) *CallsignParseResult { - result := &CallsignParseResult{ - OriginalCallsign: callsign, - ParsedTime: time.Now(), - IsValid: false, - } - - if callsign == "" { - return result - } - - // Clean and normalize the callsign - normalized := strings.TrimSpace(strings.ToUpper(callsign)) - - // Try airline pattern first (most common for commercial flights) - if matches := cm.airlinePattern.FindStringSubmatch(normalized); len(matches) == 3 { - result.AirlineCode = matches[1] - result.FlightNumber = matches[2] - result.IsValid = true - return result - } - - // Fall back to general flight pattern - if matches := cm.flightPattern.FindStringSubmatch(normalized); len(matches) == 3 { - result.AirlineCode = matches[1] - result.FlightNumber = matches[2] - result.IsValid = true - return result - } - - return result -} - -func (cm *CallsignManager) GetCallsignInfo(callsign string) (*CallsignInfo, error) { - cm.mutex.RLock() - defer cm.mutex.RUnlock() - - if callsign == "" { - return nil, fmt.Errorf("empty callsign") - } - - // First check the cache - cached, err := cm.getCallsignFromCache(callsign) - if err == nil && cached != nil { - return cached, nil - } - - // Parse the callsign - parsed := cm.ParseCallsign(callsign) - if !parsed.IsValid { - return &CallsignInfo{ - OriginalCallsign: callsign, - IsValid: false, - }, nil - } - - // Look up airline information - airline, err := cm.getAirlineByCode(parsed.AirlineCode) - if err != nil && err != sql.ErrNoRows { - return nil, fmt.Errorf("failed to lookup airline %s: %w", parsed.AirlineCode, err) - } - - // Build the result - info := &CallsignInfo{ - OriginalCallsign: callsign, - AirlineCode: parsed.AirlineCode, - FlightNumber: parsed.FlightNumber, - IsValid: true, - LastUpdated: time.Now(), - } - - if airline != nil { - info.AirlineName = airline.Name - info.AirlineCountry = airline.Country - info.DisplayName = fmt.Sprintf("%s Flight %s", airline.Name, parsed.FlightNumber) - } else { - info.DisplayName = fmt.Sprintf("%s %s", parsed.AirlineCode, parsed.FlightNumber) - } - - // Cache the result (fire and forget) - go func() { - if err := cm.cacheCallsignInfo(info); err != nil { - // Log error but don't fail the lookup - fmt.Printf("Warning: failed to cache callsign info for %s: %v\n", callsign, err) - } - }() - - return info, nil -} - -func (cm *CallsignManager) getCallsignFromCache(callsign string) (*CallsignInfo, error) { - query := ` - SELECT callsign, airline_icao, flight_number, airline_name, - airline_country, '', 1, cached_at, expires_at - FROM callsign_cache - WHERE callsign = ? AND expires_at > datetime('now') - ` - - var info CallsignInfo - var cacheExpires time.Time - - err := cm.db.QueryRow(query, callsign).Scan( - &info.OriginalCallsign, - &info.AirlineCode, - &info.FlightNumber, - &info.AirlineName, - &info.AirlineCountry, - &info.DisplayName, - &info.IsValid, - &info.LastUpdated, - &cacheExpires, - ) - - if err != nil { - return nil, err - } - - return &info, nil -} - -func (cm *CallsignManager) cacheCallsignInfo(info *CallsignInfo) error { - // Cache for 24 hours by default - cacheExpires := time.Now().Add(24 * time.Hour) - - query := ` - INSERT OR REPLACE INTO callsign_cache - (callsign, airline_icao, flight_number, airline_name, - airline_country, cached_at, expires_at) - VALUES (?, ?, ?, ?, ?, ?, ?) - ` - - _, err := cm.db.Exec(query, - info.OriginalCallsign, - info.AirlineCode, - info.FlightNumber, - info.AirlineName, - info.AirlineCountry, - info.LastUpdated, - cacheExpires, - ) - - return err -} - -func (cm *CallsignManager) getAirlineByCode(code string) (*AirlineRecord, error) { - query := ` - SELECT icao_code, iata_code, name, country, active - FROM airlines - WHERE (icao_code = ? OR iata_code = ?) AND active = 1 - ORDER BY - CASE WHEN icao_code = ? THEN 1 ELSE 2 END, - name - LIMIT 1 - ` - - var airline AirlineRecord - err := cm.db.QueryRow(query, code, code, code).Scan( - &airline.ICAOCode, - &airline.IATACode, - &airline.Name, - &airline.Country, - &airline.Active, - ) - - if err != nil { - return nil, err - } - - return &airline, nil -} - -func (cm *CallsignManager) GetAirlinesByCountry(country string) ([]AirlineRecord, error) { - cm.mutex.RLock() - defer cm.mutex.RUnlock() - - query := ` - SELECT icao_code, iata_code, name, country, active - FROM airlines - WHERE country = ? AND active = 1 - ORDER BY name - ` - - rows, err := cm.db.Query(query, country) - if err != nil { - return nil, err - } - defer rows.Close() - - var airlines []AirlineRecord - for rows.Next() { - var airline AirlineRecord - err := rows.Scan( - &airline.ICAOCode, - &airline.IATACode, - &airline.Name, - &airline.Country, - &airline.Active, - ) - if err != nil { - return nil, err - } - airlines = append(airlines, airline) - } - - return airlines, rows.Err() -} - -func (cm *CallsignManager) SearchAirlines(query string) ([]AirlineRecord, error) { - cm.mutex.RLock() - defer cm.mutex.RUnlock() - - searchQuery := ` - SELECT icao_code, iata_code, name, country, active - FROM airlines - WHERE ( - name LIKE ? OR - icao_code LIKE ? OR - iata_code LIKE ? OR - country LIKE ? - ) AND active = 1 - ORDER BY - CASE - WHEN name LIKE ? THEN 1 - WHEN icao_code = ? OR iata_code = ? THEN 2 - ELSE 3 - END, - name - LIMIT 50 - ` - - searchTerm := "%" + strings.ToUpper(query) + "%" - exactTerm := strings.ToUpper(query) - - rows, err := cm.db.Query(searchQuery, - searchTerm, searchTerm, searchTerm, searchTerm, - exactTerm, exactTerm, exactTerm, - ) - if err != nil { - return nil, err - } - defer rows.Close() - - var airlines []AirlineRecord - for rows.Next() { - var airline AirlineRecord - err := rows.Scan( - &airline.ICAOCode, - &airline.IATACode, - &airline.Name, - &airline.Country, - &airline.Active, - ) - if err != nil { - return nil, err - } - airlines = append(airlines, airline) - } - - return airlines, rows.Err() -} - -func (cm *CallsignManager) ClearExpiredCache() error { - cm.mutex.Lock() - defer cm.mutex.Unlock() - - query := `DELETE FROM callsign_cache WHERE expires_at <= datetime('now')` - _, err := cm.db.Exec(query) - return err -} - -func (cm *CallsignManager) GetCacheStats() (map[string]interface{}, error) { - cm.mutex.RLock() - defer cm.mutex.RUnlock() - - stats := make(map[string]interface{}) - - // Total cached entries - var totalCached int - err := cm.db.QueryRow(`SELECT COUNT(*) FROM callsign_cache`).Scan(&totalCached) - if err != nil { - return nil, err - } - stats["total_cached"] = totalCached - - // Valid (non-expired) entries - var validCached int - err = cm.db.QueryRow(`SELECT COUNT(*) FROM callsign_cache WHERE expires_at > datetime('now')`).Scan(&validCached) - if err != nil { - return nil, err - } - stats["valid_cached"] = validCached - - // Expired entries - stats["expired_cached"] = totalCached - validCached - - // Total airlines in database - var totalAirlines int - err = cm.db.QueryRow(`SELECT COUNT(*) FROM airlines WHERE active = 1`).Scan(&totalAirlines) - if err != nil { - return nil, err - } - stats["total_airlines"] = totalAirlines - - return stats, nil -} - -func (cm *CallsignManager) LoadEmbeddedData() error { - // Check if airlines table has data - var count int - err := cm.db.QueryRow(`SELECT COUNT(*) FROM airlines`).Scan(&count) - if err != nil { - return err - } - - if count > 0 { - // Data already loaded - return nil - } - - // For now, we'll implement this as a placeholder - // In a full implementation, this would load embedded airline data - // from embedded files or resources - return nil -} \ No newline at end of file diff --git a/internal/database/manager_history.go b/internal/database/manager_history.go deleted file mode 100644 index 1121fa3..0000000 --- a/internal/database/manager_history.go +++ /dev/null @@ -1,411 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "sync" - "time" -) - -type HistoryManager struct { - db *sql.DB - mutex sync.RWMutex - - // Configuration - maxHistoryDays int - cleanupTicker *time.Ticker - stopCleanup chan bool -} - -func NewHistoryManager(db *sql.DB, maxHistoryDays int) *HistoryManager { - hm := &HistoryManager{ - db: db, - maxHistoryDays: maxHistoryDays, - stopCleanup: make(chan bool), - } - - // Start periodic cleanup (every hour) - hm.cleanupTicker = time.NewTicker(1 * time.Hour) - go hm.periodicCleanup() - - return hm -} - -func (hm *HistoryManager) Close() { - if hm.cleanupTicker != nil { - hm.cleanupTicker.Stop() - } - if hm.stopCleanup != nil { - close(hm.stopCleanup) - } -} - -func (hm *HistoryManager) periodicCleanup() { - for { - select { - case <-hm.cleanupTicker.C: - if err := hm.CleanupOldHistory(); err != nil { - fmt.Printf("Warning: failed to cleanup old history: %v\n", err) - } - case <-hm.stopCleanup: - return - } - } -} - -func (hm *HistoryManager) RecordAircraft(record *AircraftHistoryRecord) error { - hm.mutex.Lock() - defer hm.mutex.Unlock() - - query := ` - INSERT INTO aircraft_history - (icao, callsign, squawk, latitude, longitude, altitude, - vertical_rate, speed, track, source_id, signal_strength, timestamp) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ` - - _, err := hm.db.Exec(query, - record.ICAO, - record.Callsign, - record.Squawk, - record.Latitude, - record.Longitude, - record.Altitude, - record.VerticalRate, - record.Speed, - record.Track, - record.SourceID, - record.SignalStrength, - record.Timestamp, - ) - - return err -} - -func (hm *HistoryManager) RecordAircraftBatch(records []AircraftHistoryRecord) error { - if len(records) == 0 { - return nil - } - - hm.mutex.Lock() - defer hm.mutex.Unlock() - - tx, err := hm.db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - - stmt, err := tx.Prepare(` - INSERT INTO aircraft_history - (icao, callsign, squawk, latitude, longitude, altitude, - vertical_rate, speed, track, source_id, signal_strength, timestamp) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `) - if err != nil { - return err - } - defer stmt.Close() - - for _, record := range records { - _, err := stmt.Exec( - record.ICAO, - record.Callsign, - record.Squawk, - record.Latitude, - record.Longitude, - record.Altitude, - record.VerticalRate, - record.Speed, - record.Track, - record.SourceID, - record.SignalStrength, - record.Timestamp, - ) - if err != nil { - return fmt.Errorf("failed to insert record for ICAO %s: %w", record.ICAO, err) - } - } - - return tx.Commit() -} - -func (hm *HistoryManager) GetAircraftHistory(icao string, hours int) ([]AircraftHistoryRecord, error) { - hm.mutex.RLock() - defer hm.mutex.RUnlock() - - since := time.Now().Add(-time.Duration(hours) * time.Hour) - - query := ` - SELECT icao, callsign, squawk, latitude, longitude, altitude, - vertical_rate, speed, track, source_id, signal_strength, timestamp - FROM aircraft_history - WHERE icao = ? AND timestamp >= ? - ORDER BY timestamp DESC - LIMIT 1000 - ` - - rows, err := hm.db.Query(query, icao, since) - if err != nil { - return nil, err - } - defer rows.Close() - - var records []AircraftHistoryRecord - for rows.Next() { - var record AircraftHistoryRecord - err := rows.Scan( - &record.ICAO, - &record.Callsign, - &record.Squawk, - &record.Latitude, - &record.Longitude, - &record.Altitude, - &record.VerticalRate, - &record.Speed, - &record.Track, - &record.SourceID, - &record.SignalStrength, - &record.Timestamp, - ) - if err != nil { - return nil, err - } - records = append(records, record) - } - - return records, rows.Err() -} - -func (hm *HistoryManager) GetAircraftTrack(icao string, hours int) ([]TrackPoint, error) { - hm.mutex.RLock() - defer hm.mutex.RUnlock() - - since := time.Now().Add(-time.Duration(hours) * time.Hour) - - query := ` - SELECT latitude, longitude, altitude, timestamp - FROM aircraft_history - WHERE icao = ? AND timestamp >= ? - AND latitude IS NOT NULL AND longitude IS NOT NULL - ORDER BY timestamp ASC - LIMIT 500 - ` - - rows, err := hm.db.Query(query, icao, since) - if err != nil { - return nil, err - } - defer rows.Close() - - var track []TrackPoint - for rows.Next() { - var point TrackPoint - err := rows.Scan( - &point.Latitude, - &point.Longitude, - &point.Altitude, - &point.Timestamp, - ) - if err != nil { - return nil, err - } - track = append(track, point) - } - - return track, rows.Err() -} - -func (hm *HistoryManager) GetRecentAircraft(hours int, limit int) ([]string, error) { - hm.mutex.RLock() - defer hm.mutex.RUnlock() - - since := time.Now().Add(-time.Duration(hours) * time.Hour) - - query := ` - SELECT DISTINCT icao - FROM aircraft_history - WHERE timestamp >= ? - ORDER BY MAX(timestamp) DESC - LIMIT ? - ` - - rows, err := hm.db.Query(query, since, limit) - if err != nil { - return nil, err - } - defer rows.Close() - - var icaos []string - for rows.Next() { - var icao string - err := rows.Scan(&icao) - if err != nil { - return nil, err - } - icaos = append(icaos, icao) - } - - return icaos, rows.Err() -} - -func (hm *HistoryManager) GetAircraftLastSeen(icao string) (time.Time, error) { - hm.mutex.RLock() - defer hm.mutex.RUnlock() - - query := ` - SELECT MAX(timestamp) - FROM aircraft_history - WHERE icao = ? - ` - - var lastSeen time.Time - err := hm.db.QueryRow(query, icao).Scan(&lastSeen) - return lastSeen, err -} - -func (hm *HistoryManager) CleanupOldHistory() error { - hm.mutex.Lock() - defer hm.mutex.Unlock() - - if hm.maxHistoryDays <= 0 { - return nil // No cleanup if maxHistoryDays is 0 or negative - } - - cutoff := time.Now().AddDate(0, 0, -hm.maxHistoryDays) - - query := `DELETE FROM aircraft_history WHERE timestamp < ?` - result, err := hm.db.Exec(query, cutoff) - if err != nil { - return err - } - - rowsAffected, err := result.RowsAffected() - if err == nil && rowsAffected > 0 { - fmt.Printf("Cleaned up %d old aircraft history records\n", rowsAffected) - } - - return nil -} - -func (hm *HistoryManager) GetStatistics() (map[string]interface{}, error) { - return hm.GetHistoryStats() -} - -func (hm *HistoryManager) GetHistoryStats() (map[string]interface{}, error) { - hm.mutex.RLock() - defer hm.mutex.RUnlock() - - stats := make(map[string]interface{}) - - // Total records - var totalRecords int - err := hm.db.QueryRow(`SELECT COUNT(*) FROM aircraft_history`).Scan(&totalRecords) - if err != nil { - return nil, err - } - stats["total_records"] = totalRecords - - // Unique aircraft - var uniqueAircraft int - err = hm.db.QueryRow(`SELECT COUNT(DISTINCT icao) FROM aircraft_history`).Scan(&uniqueAircraft) - if err != nil { - return nil, err - } - stats["unique_aircraft"] = uniqueAircraft - - // Recent records (last 24 hours) - var recentRecords int - since := time.Now().Add(-24 * time.Hour) - err = hm.db.QueryRow(`SELECT COUNT(*) FROM aircraft_history WHERE timestamp >= ?`, since).Scan(&recentRecords) - if err != nil { - return nil, err - } - stats["recent_records_24h"] = recentRecords - - // Oldest and newest record timestamps (only if records exist) - if totalRecords > 0 { - var oldestTimestamp, newestTimestamp time.Time - err = hm.db.QueryRow(`SELECT MIN(timestamp), MAX(timestamp) FROM aircraft_history`).Scan(&oldestTimestamp, &newestTimestamp) - if err == nil { - stats["oldest_record"] = oldestTimestamp - stats["newest_record"] = newestTimestamp - stats["history_days"] = int(time.Since(oldestTimestamp).Hours() / 24) - } - } - - return stats, nil -} - -func (hm *HistoryManager) GetActivitySummary(hours int) (map[string]interface{}, error) { - hm.mutex.RLock() - defer hm.mutex.RUnlock() - - since := time.Now().Add(-time.Duration(hours) * time.Hour) - - summary := make(map[string]interface{}) - - // Aircraft count in time period - var aircraftCount int - err := hm.db.QueryRow(` - SELECT COUNT(DISTINCT icao) - FROM aircraft_history - WHERE timestamp >= ? - `, since).Scan(&aircraftCount) - if err != nil { - return nil, err - } - summary["aircraft_count"] = aircraftCount - - // Message count in time period - var messageCount int - err = hm.db.QueryRow(` - SELECT COUNT(*) - FROM aircraft_history - WHERE timestamp >= ? - `, since).Scan(&messageCount) - if err != nil { - return nil, err - } - summary["message_count"] = messageCount - - // Most active sources - query := ` - SELECT source_id, COUNT(*) as count - FROM aircraft_history - WHERE timestamp >= ? - GROUP BY source_id - ORDER BY count DESC - LIMIT 5 - ` - - rows, err := hm.db.Query(query, since) - if err != nil { - return nil, err - } - defer rows.Close() - - sources := make([]map[string]interface{}, 0) - for rows.Next() { - var sourceID string - var count int - err := rows.Scan(&sourceID, &count) - if err != nil { - return nil, err - } - sources = append(sources, map[string]interface{}{ - "source_id": sourceID, - "count": count, - }) - } - summary["top_sources"] = sources - - return summary, nil -} - -type TrackPoint struct { - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Altitude *int `json:"altitude,omitempty"` - Timestamp time.Time `json:"timestamp"` -} \ No newline at end of file diff --git a/internal/database/migrations.go b/internal/database/migrations.go deleted file mode 100644 index e58fcbf..0000000 --- a/internal/database/migrations.go +++ /dev/null @@ -1,419 +0,0 @@ -package database - -import ( - "database/sql" - "fmt" - "sort" - "strings" - "time" -) - -// Migrator handles database schema migrations with rollback support -type Migrator struct { - conn *sql.DB -} - -// Migration represents a database schema change -type Migration struct { - Version int - Description string - Up string - Down string - DataLoss bool - Checksum string -} - -// MigrationRecord represents a completed migration in the database -type MigrationRecord struct { - Version int `json:"version"` - Description string `json:"description"` - AppliedAt time.Time `json:"applied_at"` - Checksum string `json:"checksum"` -} - -// NewMigrator creates a new database migrator -func NewMigrator(conn *sql.DB) *Migrator { - return &Migrator{conn: conn} -} - -// GetMigrations returns all available migrations in version order -func GetMigrations() []Migration { - migrations := []Migration{ - { - Version: 1, - Description: "Initial schema with aircraft history", - Up: ` - -- Schema metadata table - CREATE TABLE IF NOT EXISTS schema_info ( - version INTEGER PRIMARY KEY, - applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - description TEXT NOT NULL, - checksum TEXT NOT NULL - ); - - -- Aircraft position history - CREATE TABLE IF NOT EXISTS aircraft_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - icao TEXT NOT NULL, - timestamp TIMESTAMP NOT NULL, - latitude REAL, - longitude REAL, - altitude INTEGER, - speed INTEGER, - track INTEGER, - vertical_rate INTEGER, - squawk TEXT, - callsign TEXT, - source_id TEXT NOT NULL, - signal_strength REAL - ); - - -- Indexes for aircraft history - CREATE INDEX IF NOT EXISTS idx_aircraft_history_icao_time ON aircraft_history(icao, timestamp); - CREATE INDEX IF NOT EXISTS idx_aircraft_history_timestamp ON aircraft_history(timestamp); - CREATE INDEX IF NOT EXISTS idx_aircraft_history_callsign ON aircraft_history(callsign); - `, - Down: ` - DROP TABLE IF EXISTS aircraft_history; - DROP TABLE IF EXISTS schema_info; - `, - DataLoss: true, - }, - { - Version: 2, - Description: "Add callsign enhancement tables", - Up: ` - -- Airlines data table (unified schema for all sources) - CREATE TABLE IF NOT EXISTS airlines ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - alias TEXT, - iata_code TEXT, - icao_code TEXT, - callsign TEXT, - country TEXT, - country_code TEXT, - active BOOLEAN DEFAULT 1, - data_source TEXT NOT NULL DEFAULT 'unknown', - source_id TEXT, -- Original ID from source - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - - -- Airports data table (unified schema for all sources) - CREATE TABLE IF NOT EXISTS airports ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - ident TEXT, -- Airport identifier (ICAO, FAA, etc.) - type TEXT, -- Airport type (large_airport, medium_airport, etc.) - city TEXT, - municipality TEXT, -- More specific than city - region TEXT, -- State/province - country TEXT, - country_code TEXT, -- ISO country code - continent TEXT, - iata_code TEXT, - icao_code TEXT, - local_code TEXT, - gps_code TEXT, - latitude REAL, - longitude REAL, - elevation_ft INTEGER, - scheduled_service BOOLEAN DEFAULT 0, - home_link TEXT, - wikipedia_link TEXT, - keywords TEXT, - timezone_offset REAL, - timezone TEXT, - dst_type TEXT, - data_source TEXT NOT NULL DEFAULT 'unknown', - source_id TEXT, -- Original ID from source - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - - -- External API cache for callsign lookups - CREATE TABLE IF NOT EXISTS callsign_cache ( - callsign TEXT PRIMARY KEY, - airline_icao TEXT, - airline_iata TEXT, - airline_name TEXT, - airline_country TEXT, - flight_number TEXT, - origin_iata TEXT, - destination_iata TEXT, - aircraft_type TEXT, - route TEXT, - status TEXT, - source TEXT NOT NULL DEFAULT 'local', - cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - expires_at TIMESTAMP NOT NULL - ); - - -- Data source tracking - CREATE TABLE IF NOT EXISTS data_sources ( - name TEXT PRIMARY KEY, - license TEXT NOT NULL, - url TEXT, - version TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - record_count INTEGER DEFAULT 0, - user_accepted_license BOOLEAN DEFAULT 0 - ); - - -- Indexes for airlines - CREATE INDEX IF NOT EXISTS idx_airlines_icao_code ON airlines(icao_code); - CREATE INDEX IF NOT EXISTS idx_airlines_iata_code ON airlines(iata_code); - CREATE INDEX IF NOT EXISTS idx_airlines_callsign ON airlines(callsign); - CREATE INDEX IF NOT EXISTS idx_airlines_country_code ON airlines(country_code); - CREATE INDEX IF NOT EXISTS idx_airlines_active ON airlines(active); - CREATE INDEX IF NOT EXISTS idx_airlines_source ON airlines(data_source); - - -- Indexes for airports - CREATE INDEX IF NOT EXISTS idx_airports_icao_code ON airports(icao_code); - CREATE INDEX IF NOT EXISTS idx_airports_iata_code ON airports(iata_code); - CREATE INDEX IF NOT EXISTS idx_airports_ident ON airports(ident); - CREATE INDEX IF NOT EXISTS idx_airports_country_code ON airports(country_code); - CREATE INDEX IF NOT EXISTS idx_airports_type ON airports(type); - CREATE INDEX IF NOT EXISTS idx_airports_coords ON airports(latitude, longitude); - CREATE INDEX IF NOT EXISTS idx_airports_source ON airports(data_source); - - -- Indexes for callsign cache - CREATE INDEX IF NOT EXISTS idx_callsign_cache_expires ON callsign_cache(expires_at); - CREATE INDEX IF NOT EXISTS idx_callsign_cache_airline ON callsign_cache(airline_icao); - `, - Down: ` - DROP TABLE IF EXISTS callsign_cache; - DROP TABLE IF EXISTS airports; - DROP TABLE IF EXISTS airlines; - DROP TABLE IF EXISTS data_sources; - `, - DataLoss: true, - }, - // Future migrations will be added here - } - - // Calculate checksums - for i := range migrations { - migrations[i].Checksum = calculateChecksum(migrations[i].Up) - } - - // Sort by version - sort.Slice(migrations, func(i, j int) bool { - return migrations[i].Version < migrations[j].Version - }) - - return migrations -} - -// MigrateToLatest runs all pending migrations to bring database to latest schema -func (m *Migrator) MigrateToLatest() error { - currentVersion, err := m.getCurrentVersion() - if err != nil { - return fmt.Errorf("failed to get current version: %v", err) - } - - migrations := GetMigrations() - - for _, migration := range migrations { - if migration.Version <= currentVersion { - continue - } - - if err := m.applyMigration(migration); err != nil { - return fmt.Errorf("failed to apply migration %d: %v", migration.Version, err) - } - } - - return nil -} - -// MigrateTo runs migrations to reach a specific version -func (m *Migrator) MigrateTo(targetVersion int) error { - currentVersion, err := m.getCurrentVersion() - if err != nil { - return fmt.Errorf("failed to get current version: %v", err) - } - - if targetVersion == currentVersion { - return nil // Already at target version - } - - migrations := GetMigrations() - - if targetVersion > currentVersion { - // Forward migration - for _, migration := range migrations { - if migration.Version <= currentVersion || migration.Version > targetVersion { - continue - } - - if err := m.applyMigration(migration); err != nil { - return fmt.Errorf("failed to apply migration %d: %v", migration.Version, err) - } - } - } else { - // Rollback migration - // Sort in reverse order for rollback - sort.Slice(migrations, func(i, j int) bool { - return migrations[i].Version > migrations[j].Version - }) - - for _, migration := range migrations { - if migration.Version > currentVersion || migration.Version <= targetVersion { - continue - } - - if err := m.rollbackMigration(migration); err != nil { - return fmt.Errorf("failed to rollback migration %d: %v", migration.Version, err) - } - } - } - - return nil -} - -// GetAppliedMigrations returns all migrations that have been applied -func (m *Migrator) GetAppliedMigrations() ([]MigrationRecord, error) { - // Ensure schema_info table exists - if err := m.ensureSchemaInfoTable(); err != nil { - return nil, err - } - - query := ` - SELECT version, description, applied_at, checksum - FROM schema_info - ORDER BY version - ` - - rows, err := m.conn.Query(query) - if err != nil { - return nil, fmt.Errorf("failed to query applied migrations: %v", err) - } - defer rows.Close() - - var migrations []MigrationRecord - for rows.Next() { - var migration MigrationRecord - err := rows.Scan( - &migration.Version, - &migration.Description, - &migration.AppliedAt, - &migration.Checksum, - ) - if err != nil { - return nil, fmt.Errorf("failed to scan migration record: %v", err) - } - migrations = append(migrations, migration) - } - - return migrations, nil -} - -// getCurrentVersion returns the highest applied migration version -func (m *Migrator) getCurrentVersion() (int, error) { - if err := m.ensureSchemaInfoTable(); err != nil { - return 0, err - } - - var version int - err := m.conn.QueryRow(`SELECT COALESCE(MAX(version), 0) FROM schema_info`).Scan(&version) - if err != nil { - return 0, fmt.Errorf("failed to get current version: %v", err) - } - - return version, nil -} - -// applyMigration executes a migration in a transaction -func (m *Migrator) applyMigration(migration Migration) error { - tx, err := m.conn.Begin() - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Warn about data loss - if migration.DataLoss { - // In a real application, this would show a warning to the user - // For now, we'll just log it - } - - // Execute migration SQL - statements := strings.Split(migration.Up, ";") - for _, stmt := range statements { - stmt = strings.TrimSpace(stmt) - if stmt == "" { - continue - } - - if _, err := tx.Exec(stmt); err != nil { - return fmt.Errorf("failed to execute migration statement: %v", err) - } - } - - // Record migration - _, err = tx.Exec(` - INSERT INTO schema_info (version, description, checksum) - VALUES (?, ?, ?) - `, migration.Version, migration.Description, migration.Checksum) - - if err != nil { - return fmt.Errorf("failed to record migration: %v", err) - } - - return tx.Commit() -} - -// rollbackMigration executes a migration rollback in a transaction -func (m *Migrator) rollbackMigration(migration Migration) error { - if migration.Down == "" { - return fmt.Errorf("migration %d has no rollback script", migration.Version) - } - - tx, err := m.conn.Begin() - if err != nil { - return fmt.Errorf("failed to begin transaction: %v", err) - } - defer tx.Rollback() - - // Execute rollback SQL - statements := strings.Split(migration.Down, ";") - for _, stmt := range statements { - stmt = strings.TrimSpace(stmt) - if stmt == "" { - continue - } - - if _, err := tx.Exec(stmt); err != nil { - return fmt.Errorf("failed to execute rollback statement: %v", err) - } - } - - // Remove migration record - _, err = tx.Exec(`DELETE FROM schema_info WHERE version = ?`, migration.Version) - if err != nil { - return fmt.Errorf("failed to remove migration record: %v", err) - } - - return tx.Commit() -} - -// ensureSchemaInfoTable creates the schema_info table if it doesn't exist -func (m *Migrator) ensureSchemaInfoTable() error { - _, err := m.conn.Exec(` - CREATE TABLE IF NOT EXISTS schema_info ( - version INTEGER PRIMARY KEY, - applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - description TEXT NOT NULL, - checksum TEXT NOT NULL - ) - `) - return err -} - -// calculateChecksum generates a checksum for migration content -func calculateChecksum(content string) string { - // Simple checksum - in production, use a proper hash function - return fmt.Sprintf("%x", len(content)) -} \ No newline at end of file diff --git a/internal/database/path.go b/internal/database/path.go deleted file mode 100644 index 64d98fc..0000000 --- a/internal/database/path.go +++ /dev/null @@ -1,174 +0,0 @@ -package database - -import ( - "fmt" - "os" - "path/filepath" - "runtime" -) - -// ResolveDatabasePath determines the appropriate database file location -// based on configuration, system type, and available permissions -func ResolveDatabasePath(configPath string) (string, error) { - // Use explicit configuration path if provided - if configPath != "" { - if err := ensureDirExists(filepath.Dir(configPath)); err != nil { - return "", fmt.Errorf("cannot create directory for configured path %s: %v", configPath, err) - } - return configPath, nil - } - - // Try system location first (for services) - if systemPath, err := trySystemPath(); err == nil { - return systemPath, nil - } - - // Try user data directory - if userPath, err := tryUserPath(); err == nil { - return userPath, nil - } - - // Fallback to current directory - return tryCurrentDirPath() -} - -// trySystemPath attempts to use system-wide database location -func trySystemPath() (string, error) { - var systemDir string - - switch runtime.GOOS { - case "linux": - systemDir = "/var/lib/skyview" - case "darwin": - systemDir = "/usr/local/var/skyview" - case "windows": - systemDir = filepath.Join(os.Getenv("PROGRAMDATA"), "skyview") - default: - return "", fmt.Errorf("system path not supported on %s", runtime.GOOS) - } - - // Check if directory exists and is writable - if err := ensureDirExists(systemDir); err != nil { - return "", err - } - - dbPath := filepath.Join(systemDir, "skyview.db") - - // Test write permissions - if err := testWritePermissions(dbPath); err != nil { - return "", err - } - - return dbPath, nil -} - -// tryUserPath attempts to use user data directory -func tryUserPath() (string, error) { - var userDataDir string - - switch runtime.GOOS { - case "linux": - if xdgData := os.Getenv("XDG_DATA_HOME"); xdgData != "" { - userDataDir = xdgData - } else { - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - userDataDir = filepath.Join(home, ".local", "share") - } - case "darwin": - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - userDataDir = filepath.Join(home, "Library", "Application Support") - case "windows": - userDataDir = os.Getenv("APPDATA") - if userDataDir == "" { - return "", fmt.Errorf("APPDATA environment variable not set") - } - default: - return "", fmt.Errorf("user path not supported on %s", runtime.GOOS) - } - - skyviewDir := filepath.Join(userDataDir, "skyview") - - if err := ensureDirExists(skyviewDir); err != nil { - return "", err - } - - dbPath := filepath.Join(skyviewDir, "skyview.db") - - // Test write permissions - if err := testWritePermissions(dbPath); err != nil { - return "", err - } - - return dbPath, nil -} - -// tryCurrentDirPath uses current directory as fallback -func tryCurrentDirPath() (string, error) { - currentDir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("cannot get current directory: %v", err) - } - - dbPath := filepath.Join(currentDir, "skyview.db") - - // Test write permissions - if err := testWritePermissions(dbPath); err != nil { - return "", err - } - - return dbPath, nil -} - -// ensureDirExists creates directory if it doesn't exist -func ensureDirExists(dir string) error { - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("cannot create directory %s: %v", dir, err) - } - } else if err != nil { - return fmt.Errorf("cannot access directory %s: %v", dir, err) - } - - return nil -} - -// testWritePermissions verifies write access to the database path -func testWritePermissions(dbPath string) error { - dir := filepath.Dir(dbPath) - - // Check directory write permissions - testFile := filepath.Join(dir, ".skyview_write_test") - if err := os.WriteFile(testFile, []byte("test"), 0644); err != nil { - return fmt.Errorf("no write permission to directory %s: %v", dir, err) - } - - // Clean up test file - os.Remove(testFile) - - return nil -} - -// GetDatabaseDirectory returns the directory containing the database file -func GetDatabaseDirectory(dbPath string) string { - return filepath.Dir(dbPath) -} - -// IsSystemPath returns true if the database path is in a system location -func IsSystemPath(dbPath string) bool { - switch runtime.GOOS { - case "linux": - return filepath.HasPrefix(dbPath, "/var/lib/skyview") - case "darwin": - return filepath.HasPrefix(dbPath, "/usr/local/var/skyview") - case "windows": - programData := os.Getenv("PROGRAMDATA") - return programData != "" && filepath.HasPrefix(dbPath, filepath.Join(programData, "skyview")) - } - return false -} \ No newline at end of file diff --git a/internal/merger/merger.go b/internal/merger/merger.go index 6288a37..5fe3a80 100644 --- a/internal/merger/merger.go +++ b/internal/merger/merger.go @@ -27,7 +27,6 @@ import ( "sync" "time" - "skyview/internal/database" "skyview/internal/icao" "skyview/internal/modes" "skyview/internal/squawk" @@ -273,7 +272,6 @@ type Merger struct { sources map[string]*Source // Source ID -> source information icaoDB *icao.Database // ICAO country lookup database squawkDB *squawk.Database // Transponder code lookup database - db *database.Database // Optional persistent database mu sync.RWMutex // Protects all maps and slices historyLimit int // Maximum history points to retain staleTimeout time.Duration // Time before aircraft considered stale (15 seconds) @@ -297,23 +295,6 @@ type updateMetric struct { // // The merger is ready for immediate use after creation. func NewMerger() (*Merger, error) { - return NewMergerWithDatabase(nil) -} - -// NewMergerWithDatabase creates a new aircraft data merger with optional database support. -// -// If a database is provided, aircraft positions will be persisted to the database -// for historical analysis and long-term tracking. The database parameter can be nil -// to disable persistence. -// -// Default settings: -// - History limit: 500 points per aircraft -// - Stale timeout: 15 seconds -// - Empty aircraft and source maps -// - Update metrics tracking enabled -// -// The merger is ready for immediate use after creation. -func NewMergerWithDatabase(db *database.Database) (*Merger, error) { icaoDB, err := icao.NewDatabase() if err != nil { return nil, fmt.Errorf("failed to initialize ICAO database: %w", err) @@ -326,7 +307,6 @@ func NewMergerWithDatabase(db *database.Database) (*Merger, error) { sources: make(map[string]*Source), icaoDB: icaoDB, squawkDB: squawkDB, - db: db, historyLimit: 500, staleTimeout: 15 * time.Second, // Aircraft timeout - reasonable for ADS-B tracking updateMetrics: make(map[uint32]*updateMetric), @@ -448,11 +428,6 @@ func (m *Merger) UpdateAircraft(sourceID string, aircraft *modes.Aircraft, signa state.LastUpdate = timestamp state.TotalMessages++ - - // Persist to database if available and aircraft has position - if m.db != nil && aircraft.Latitude != 0 && aircraft.Longitude != 0 { - m.saveAircraftToDatabase(aircraft, sourceID, signal, timestamp) - } } // mergeAircraftData intelligently merges data from multiple sources with conflict resolution. @@ -1073,49 +1048,6 @@ func (m *Merger) validatePosition(aircraft *modes.Aircraft, state *AircraftState return result } -// saveAircraftToDatabase persists aircraft position data to the database -func (m *Merger) saveAircraftToDatabase(aircraft *modes.Aircraft, sourceID string, signal float64, timestamp time.Time) { - // Convert ICAO24 to hex string - icaoHex := fmt.Sprintf("%06X", aircraft.ICAO24) - - // Prepare database record - record := database.AircraftHistoryRecord{ - ICAO: icaoHex, - Timestamp: timestamp, - Latitude: &aircraft.Latitude, - Longitude: &aircraft.Longitude, - SourceID: sourceID, - SignalStrength: &signal, - } - - // Add optional fields if available - if aircraft.Altitude > 0 { - record.Altitude = &aircraft.Altitude - } - if aircraft.GroundSpeed > 0 { - record.Speed = &aircraft.GroundSpeed - } - if aircraft.Track >= 0 && aircraft.Track < 360 { - record.Track = &aircraft.Track - } - if aircraft.VerticalRate != 0 { - record.VerticalRate = &aircraft.VerticalRate - } - if aircraft.Squawk != "" && aircraft.Squawk != "0000" { - record.Squawk = &aircraft.Squawk - } - if aircraft.Callsign != "" { - record.Callsign = &aircraft.Callsign - } - - // Save to database (non-blocking to avoid slowing down real-time processing) - go func() { - if err := m.db.GetHistoryManager().RecordAircraft(&record); err != nil { - log.Printf("Warning: Failed to save aircraft %s to database: %v", icaoHex, err) - } - }() -} - // Close closes the merger and releases resources func (m *Merger) Close() error { m.mu.Lock() diff --git a/internal/server/server.go b/internal/server/server.go index b2f54e4..c03cd7b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -18,10 +18,8 @@ import ( "embed" "encoding/json" "fmt" - "io/fs" "log" "net/http" - "os" "path" "strconv" "strings" @@ -31,7 +29,6 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/websocket" - "skyview/internal/database" "skyview/internal/merger" ) @@ -55,13 +52,12 @@ type OriginConfig struct { // - Concurrent broadcast system for WebSocket clients // - CORS support for cross-origin web applications type Server struct { - host string // Bind address for HTTP server - port int // TCP port for HTTP server - merger *merger.Merger // Data source for aircraft information - database *database.Database // Optional database for persistence - staticFiles embed.FS // Embedded static web assets - server *http.Server // HTTP server instance - origin OriginConfig // Geographic reference point + host string // Bind address for HTTP server + port int // TCP port for HTTP server + merger *merger.Merger // Data source for aircraft information + staticFiles embed.FS // Embedded static web assets + server *http.Server // HTTP server instance + origin OriginConfig // Geographic reference point // WebSocket management wsClients map[*websocket.Conn]bool // Active WebSocket client connections @@ -102,17 +98,15 @@ type AircraftUpdate struct { // - host: Bind address (empty for all interfaces, "localhost" for local only) // - port: TCP port number for the HTTP server // - merger: Data merger instance providing aircraft information -// - database: Optional database for persistence and callsign enhancement // - staticFiles: Embedded filesystem containing web assets // - origin: Geographic reference point for the map interface // // Returns a configured but not yet started server instance. -func NewWebServer(host string, port int, merger *merger.Merger, database *database.Database, staticFiles embed.FS, origin OriginConfig) *Server { +func NewWebServer(host string, port int, merger *merger.Merger, staticFiles embed.FS, origin OriginConfig) *Server { return &Server{ host: host, port: port, merger: merger, - database: database, staticFiles: staticFiles, origin: origin, wsClients: make(map[*websocket.Conn]bool), @@ -210,10 +204,6 @@ func (s *Server) setupRoutes() http.Handler { api.HandleFunc("/origin", s.handleGetOrigin).Methods("GET") api.HandleFunc("/coverage/{sourceId}", s.handleGetCoverage).Methods("GET") api.HandleFunc("/heatmap/{sourceId}", s.handleGetHeatmap).Methods("GET") - // Database API endpoints - api.HandleFunc("/database/status", s.handleGetDatabaseStatus).Methods("GET") - api.HandleFunc("/database/sources", s.handleGetDataSources).Methods("GET") - api.HandleFunc("/callsign/{callsign}", s.handleGetCallsignInfo).Methods("GET") // WebSocket router.HandleFunc("/ws", s.handleWebSocket) @@ -224,8 +214,6 @@ func (s *Server) setupRoutes() http.Handler { // Main page router.HandleFunc("/", s.handleIndex) - // Database status page - router.HandleFunc("/database", s.handleDatabasePage) // Enable CORS return s.enableCORS(router) @@ -910,193 +898,3 @@ func (s *Server) handleDebugWebSocket(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(response) } - -// handleGetDatabaseStatus returns database status and statistics -func (s *Server) handleGetDatabaseStatus(w http.ResponseWriter, r *http.Request) { - if s.database == nil { - http.Error(w, "Database not available", http.StatusServiceUnavailable) - return - } - - response := make(map[string]interface{}) - - // Get database path and size information - dbConfig := s.database.GetConfig() - dbPath := dbConfig.Path - response["path"] = dbPath - - // Get file size and modification time - if stat, err := os.Stat(dbPath); err == nil { - response["size_bytes"] = stat.Size() - response["size_mb"] = float64(stat.Size()) / (1024 * 1024) - response["modified"] = stat.ModTime().Unix() - } - - // Get optimization statistics - optimizer := database.NewOptimizationManager(s.database, dbConfig) - if optimizationStats, err := optimizer.GetOptimizationStats(); err == nil { - response["efficiency_percent"] = optimizationStats.Efficiency - response["page_size"] = optimizationStats.PageSize - response["page_count"] = optimizationStats.PageCount - response["used_pages"] = optimizationStats.UsedPages - response["free_pages"] = optimizationStats.FreePages - response["auto_vacuum_enabled"] = optimizationStats.AutoVacuumEnabled - if !optimizationStats.LastVacuum.IsZero() { - response["last_vacuum"] = optimizationStats.LastVacuum.Unix() - } - } - - // Get history statistics - historyStats, err := s.database.GetHistoryManager().GetStatistics() - if err != nil { - log.Printf("Error getting history statistics: %v", err) - historyStats = make(map[string]interface{}) - } - - // Get callsign statistics if available - callsignStats := make(map[string]interface{}) - if callsignManager := s.database.GetCallsignManager(); callsignManager != nil { - stats, err := callsignManager.GetCacheStats() - if err != nil { - log.Printf("Error getting callsign statistics: %v", err) - } else { - callsignStats = stats - } - } - - // Get record counts for reference data - var airportCount, airlineCount int - s.database.GetConnection().QueryRow(`SELECT COUNT(*) FROM airports`).Scan(&airportCount) - s.database.GetConnection().QueryRow(`SELECT COUNT(*) FROM airlines`).Scan(&airlineCount) - - referenceData := make(map[string]interface{}) - referenceData["airports"] = airportCount - referenceData["airlines"] = airlineCount - - response["database_available"] = true - response["path"] = dbPath - response["reference_data"] = referenceData - response["history"] = historyStats - response["callsign"] = callsignStats - response["timestamp"] = time.Now().Unix() - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) -} - -// handleGetDataSources returns information about loaded external data sources -func (s *Server) handleGetDataSources(w http.ResponseWriter, r *http.Request) { - if s.database == nil { - http.Error(w, "Database not available", http.StatusServiceUnavailable) - return - } - - // Create data loader instance - loader := database.NewDataLoader(s.database.GetConnection()) - - availableSources := database.GetAvailableDataSources() - loadedSources, err := loader.GetLoadedDataSources() - if err != nil { - log.Printf("Error getting loaded data sources: %v", err) - loadedSources = []database.DataSource{} - } - - response := map[string]interface{}{ - "available": availableSources, - "loaded": loadedSources, - "timestamp": time.Now().Unix(), - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) -} - -// handleGetCallsignInfo returns enriched information about a callsign -func (s *Server) handleGetCallsignInfo(w http.ResponseWriter, r *http.Request) { - if s.database == nil { - http.Error(w, "Database not available", http.StatusServiceUnavailable) - return - } - - // Extract callsign from URL parameters - vars := mux.Vars(r) - callsign := vars["callsign"] - - if callsign == "" { - http.Error(w, "Callsign parameter required", http.StatusBadRequest) - return - } - - // Get callsign information from database - callsignInfo, err := s.database.GetCallsignManager().GetCallsignInfo(callsign) - if err != nil { - log.Printf("Error getting callsign info for %s: %v", callsign, err) - http.Error(w, "Failed to lookup callsign information", http.StatusInternalServerError) - return - } - - response := map[string]interface{}{ - "callsign": callsignInfo, - "timestamp": time.Now().Unix(), - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) -} - -// debugEmbeddedFiles lists all embedded files for debugging -func (s *Server) debugEmbeddedFiles() { - log.Println("=== Debugging Embedded Files ===") - err := fs.WalkDir(s.staticFiles, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - log.Printf("Error walking %s: %v", path, err) - return nil - } - if !d.IsDir() { - info, _ := d.Info() - log.Printf("Embedded file: %s (size: %d bytes)", path, info.Size()) - } else { - log.Printf("Embedded dir: %s/", path) - } - return nil - }) - if err != nil { - log.Printf("Error walking embedded files: %v", err) - } - log.Println("=== End Embedded Files Debug ===") -} - -// handleDatabasePage serves the database status page -func (s *Server) handleDatabasePage(w http.ResponseWriter, r *http.Request) { - // Debug embedded files first - s.debugEmbeddedFiles() - - // Try to read the database HTML file from embedded assets - data, err := s.staticFiles.ReadFile("static/database.html") - if err != nil { - log.Printf("Error reading database.html: %v", err) - - // Fallback: serve a simple HTML page with API calls - fallbackHTML := ` -Database Status - SkyView - -

Database Status

-
Loading...
- -` - - w.Header().Set("Content-Type", "text/html") - w.Write([]byte(fallbackHTML)) - return - } - - w.Header().Set("Content-Type", "text/html") - w.Write(data) -} diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh index 6704674..e22f6e2 100755 --- a/scripts/build-deb.sh +++ b/scripts/build-deb.sh @@ -61,24 +61,13 @@ if ! go build -ldflags="$LDFLAGS" \ exit 1 fi -# Build skyview-data utility -echo_info "Building skyview-data..." -if ! go build -ldflags="$LDFLAGS" \ - -o "$DEB_DIR/usr/bin/skyview-data" \ - ./cmd/skyview-data; then - echo_error "Failed to build skyview-data" - exit 1 -fi - echo_info "Built binaries:" echo_info " skyview: $(file "$DEB_DIR/usr/bin/skyview")" -echo_info " beast-dump: $(file "$DEB_DIR/usr/bin/beast-dump")" -echo_info " skyview-data: $(file "$DEB_DIR/usr/bin/skyview-data")" +echo_info " beast-dump: $(file "$DEB_DIR/usr/bin/beast-dump")" # Set executable permissions chmod +x "$DEB_DIR/usr/bin/skyview" chmod +x "$DEB_DIR/usr/bin/beast-dump" -chmod +x "$DEB_DIR/usr/bin/skyview-data" # Generate incremental changelog from git history echo_info "Generating incremental changelog from git history..." diff --git a/scripts/update-database.sh b/scripts/update-database.sh deleted file mode 100755 index 670e456..0000000 --- a/scripts/update-database.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -# SkyView Database Auto-Update Script -# Safe for cron execution - updates only public domain sources - -set -e - -# Configuration -SKYVIEW_DATA_CMD="${SKYVIEW_DATA_CMD:-skyview-data}" -LOCK_FILE="${TMPDIR:-/tmp}/skyview-data-update.lock" -LOG_FILE="${LOG_FILE:-/var/log/skyview/database-update.log}" - -# Colors for output (disabled if not a tty) -if [ -t 1 ]; then - RED='\033[0;31m' - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - NC='\033[0m' -else - RED='' - GREEN='' - YELLOW='' - NC='' -fi - -log() { - echo "$(date '+%Y-%m-%d %H:%M:%S') $1" | tee -a "${LOG_FILE}" 2>/dev/null || echo "$(date '+%Y-%m-%d %H:%M:%S') $1" -} - -error() { - echo -e "${RED}ERROR: $1${NC}" >&2 - log "ERROR: $1" -} - -success() { - echo -e "${GREEN}$1${NC}" - log "$1" -} - -warn() { - echo -e "${YELLOW}WARNING: $1${NC}" - log "WARNING: $1" -} - -# Check for lock file (prevent concurrent runs) -if [ -f "$LOCK_FILE" ]; then - if kill -0 "$(cat "$LOCK_FILE")" 2>/dev/null; then - error "Another instance is already running (PID: $(cat "$LOCK_FILE"))" - exit 1 - else - warn "Removing stale lock file" - rm -f "$LOCK_FILE" - fi -fi - -# Create lock file -echo $$ > "$LOCK_FILE" -trap 'rm -f "$LOCK_FILE"' EXIT - -log "Starting SkyView database update" - -# Check if skyview-data command exists -if ! command -v "$SKYVIEW_DATA_CMD" >/dev/null 2>&1; then - error "skyview-data command not found in PATH" - exit 1 -fi - -# Update database (this will auto-initialize if needed) -log "Running: $SKYVIEW_DATA_CMD update" -if "$SKYVIEW_DATA_CMD" update; then - success "Database update completed successfully" - - # Show status for logging - "$SKYVIEW_DATA_CMD" status 2>/dev/null | while IFS= read -r line; do - log "STATUS: $line" - done - - exit 0 -else - error "Database update failed" - exit 1 -fi \ No newline at end of file