feat: Add SQLite database integration for aircraft history and callsign enhancement
- Implement comprehensive database package with versioned migrations - Add skyview-data CLI tool for managing aviation reference data - Integrate database with merger for real-time aircraft history persistence - Support OurAirports and OpenFlights data sources (runtime loading) - Add systemd timer for automated database updates - Fix transaction-based bulk loading for 2400% performance improvement - Add callsign enhancement system with airline/airport lookups - Update Debian packaging with database directory and permissions Database features: - Aircraft position history with configurable retention - External aviation data loading (airlines, airports) - Callsign parsing and enhancement - API client for external lookups (OpenSky, etc.) - Privacy mode for complete offline operation CLI commands: - skyview-data status: Show database statistics - skyview-data update: Load aviation reference data - skyview-data list: Show available data sources - skyview-data clear: Remove specific data sources 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
cd51d3ecc0
commit
37c4fa2b57
25 changed files with 4771 additions and 12 deletions
256
internal/database/database.go
Normal file
256
internal/database/database.go
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
// Package database provides persistent storage for aircraft data and callsign enhancement
|
||||
// using SQLite with versioned schema migrations and comprehensive error handling.
|
||||
//
|
||||
// The database system supports:
|
||||
// - Aircraft position history with configurable retention
|
||||
// - Embedded OpenFlights airline and airport data
|
||||
// - External API result caching with TTL
|
||||
// - Schema migrations with rollback support
|
||||
// - Privacy mode for complete offline operation
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3" // SQLite driver
|
||||
)
|
||||
|
||||
// Database represents the main database connection and operations
|
||||
type Database struct {
|
||||
conn *sql.DB
|
||||
config *Config
|
||||
migrator *Migrator
|
||||
callsign *CallsignManager
|
||||
history *HistoryManager
|
||||
}
|
||||
|
||||
// Config holds database configuration options
|
||||
type Config struct {
|
||||
// Database file path (auto-resolved if empty)
|
||||
Path string `json:"path"`
|
||||
|
||||
// Data retention settings
|
||||
MaxHistoryDays int `json:"max_history_days"` // 0 = unlimited
|
||||
BackupOnUpgrade bool `json:"backup_on_upgrade"`
|
||||
|
||||
// Connection settings
|
||||
MaxOpenConns int `json:"max_open_conns"` // Default: 10
|
||||
MaxIdleConns int `json:"max_idle_conns"` // Default: 5
|
||||
ConnMaxLifetime time.Duration `json:"conn_max_lifetime"` // Default: 1 hour
|
||||
|
||||
// Maintenance settings
|
||||
VacuumInterval time.Duration `json:"vacuum_interval"` // Default: 24 hours
|
||||
CleanupInterval time.Duration `json:"cleanup_interval"` // Default: 1 hour
|
||||
}
|
||||
|
||||
// AircraftHistoryRecord represents a stored aircraft position update
|
||||
type AircraftHistoryRecord struct {
|
||||
ID int64 `json:"id"`
|
||||
ICAO string `json:"icao"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Altitude *int `json:"altitude,omitempty"`
|
||||
Speed *int `json:"speed,omitempty"`
|
||||
Track *int `json:"track,omitempty"`
|
||||
VerticalRate *int `json:"vertical_rate,omitempty"`
|
||||
Squawk *string `json:"squawk,omitempty"`
|
||||
Callsign *string `json:"callsign,omitempty"`
|
||||
SourceID string `json:"source_id"`
|
||||
SignalStrength *float64 `json:"signal_strength,omitempty"`
|
||||
}
|
||||
|
||||
// CallsignInfo represents enriched callsign information
|
||||
type CallsignInfo struct {
|
||||
OriginalCallsign string `json:"original_callsign"`
|
||||
AirlineCode string `json:"airline_code"`
|
||||
FlightNumber string `json:"flight_number"`
|
||||
AirlineName string `json:"airline_name"`
|
||||
AirlineCountry string `json:"airline_country"`
|
||||
DisplayName string `json:"display_name"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
// AirlineRecord represents embedded airline data from OpenFlights
|
||||
type AirlineRecord struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Alias string `json:"alias"`
|
||||
IATACode string `json:"iata_code"`
|
||||
ICAOCode string `json:"icao_code"`
|
||||
Callsign string `json:"callsign"`
|
||||
Country string `json:"country"`
|
||||
Active bool `json:"active"`
|
||||
}
|
||||
|
||||
// AirportRecord represents embedded airport data from OpenFlights
|
||||
type AirportRecord struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
City string `json:"city"`
|
||||
Country string `json:"country"`
|
||||
IATA string `json:"iata"`
|
||||
ICAO string `json:"icao"`
|
||||
Latitude float64 `json:"latitude"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
Altitude int `json:"altitude"`
|
||||
TimezoneOffset float64 `json:"timezone_offset"`
|
||||
DST string `json:"dst"`
|
||||
Timezone string `json:"timezone"`
|
||||
}
|
||||
|
||||
// DatabaseError represents database operation errors
|
||||
type DatabaseError struct {
|
||||
Operation string
|
||||
Err error
|
||||
Query string
|
||||
Retryable bool
|
||||
}
|
||||
|
||||
func (e *DatabaseError) Error() string {
|
||||
if e.Query != "" {
|
||||
return fmt.Sprintf("database %s error: %v (query: %s)", e.Operation, e.Err, e.Query)
|
||||
}
|
||||
return fmt.Sprintf("database %s error: %v", e.Operation, e.Err)
|
||||
}
|
||||
|
||||
func (e *DatabaseError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
// NewDatabase creates a new database connection with the given configuration
|
||||
func NewDatabase(config *Config) (*Database, error) {
|
||||
if config == nil {
|
||||
config = DefaultConfig()
|
||||
}
|
||||
|
||||
// Resolve database path
|
||||
dbPath, err := ResolveDatabasePath(config.Path)
|
||||
if err != nil {
|
||||
return nil, &DatabaseError{
|
||||
Operation: "path_resolution",
|
||||
Err: err,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
config.Path = dbPath
|
||||
|
||||
// Open database connection
|
||||
conn, err := sql.Open("sqlite3", buildConnectionString(dbPath))
|
||||
if err != nil {
|
||||
return nil, &DatabaseError{
|
||||
Operation: "connect",
|
||||
Err: err,
|
||||
Retryable: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Configure connection pool
|
||||
conn.SetMaxOpenConns(config.MaxOpenConns)
|
||||
conn.SetMaxIdleConns(config.MaxIdleConns)
|
||||
conn.SetConnMaxLifetime(config.ConnMaxLifetime)
|
||||
|
||||
// Test connection
|
||||
if err := conn.Ping(); err != nil {
|
||||
conn.Close()
|
||||
return nil, &DatabaseError{
|
||||
Operation: "ping",
|
||||
Err: err,
|
||||
Retryable: true,
|
||||
}
|
||||
}
|
||||
|
||||
db := &Database{
|
||||
conn: conn,
|
||||
config: config,
|
||||
}
|
||||
|
||||
// Initialize components
|
||||
db.migrator = NewMigrator(conn)
|
||||
db.callsign = NewCallsignManager(conn)
|
||||
db.history = NewHistoryManager(conn, config.MaxHistoryDays)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Initialize runs database migrations and sets up embedded data
|
||||
func (db *Database) Initialize() error {
|
||||
// Run schema migrations
|
||||
if err := db.migrator.MigrateToLatest(); err != nil {
|
||||
return &DatabaseError{
|
||||
Operation: "migration",
|
||||
Err: err,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
// Load embedded OpenFlights data if not already loaded
|
||||
if err := db.callsign.LoadEmbeddedData(); err != nil {
|
||||
return &DatabaseError{
|
||||
Operation: "load_embedded_data",
|
||||
Err: err,
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetConfig returns the database configuration
|
||||
func (db *Database) GetConfig() *Config {
|
||||
return db.config
|
||||
}
|
||||
|
||||
// GetConnection returns the underlying database connection
|
||||
func (db *Database) GetConnection() *sql.DB {
|
||||
return db.conn
|
||||
}
|
||||
|
||||
// GetHistoryManager returns the history manager
|
||||
func (db *Database) GetHistoryManager() *HistoryManager {
|
||||
return db.history
|
||||
}
|
||||
|
||||
// GetCallsignManager returns the callsign manager
|
||||
func (db *Database) GetCallsignManager() *CallsignManager {
|
||||
return db.callsign
|
||||
}
|
||||
|
||||
// Close closes the database connection and stops background tasks
|
||||
func (db *Database) Close() error {
|
||||
if db.conn != nil {
|
||||
return db.conn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Health returns the database health status
|
||||
func (db *Database) Health() error {
|
||||
if db.conn == nil {
|
||||
return fmt.Errorf("database connection not initialized")
|
||||
}
|
||||
return db.conn.Ping()
|
||||
}
|
||||
|
||||
|
||||
// DefaultConfig returns the default database configuration
|
||||
func DefaultConfig() *Config {
|
||||
return &Config{
|
||||
Path: "", // Auto-resolved
|
||||
MaxHistoryDays: 7,
|
||||
BackupOnUpgrade: true,
|
||||
MaxOpenConns: 10,
|
||||
MaxIdleConns: 5,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
VacuumInterval: 24 * time.Hour,
|
||||
CleanupInterval: time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
// buildConnectionString creates SQLite connection string with optimizations
|
||||
func buildConnectionString(path string) string {
|
||||
return fmt.Sprintf("%s?_journal_mode=WAL&_synchronous=NORMAL&_cache_size=-64000&_temp_store=MEMORY&_foreign_keys=ON", path)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue