feat: Add SQLite database integration for aircraft history and callsign enhancement
- Implement comprehensive database package with versioned migrations - Add skyview-data CLI tool for managing aviation reference data - Integrate database with merger for real-time aircraft history persistence - Support OurAirports and OpenFlights data sources (runtime loading) - Add systemd timer for automated database updates - Fix transaction-based bulk loading for 2400% performance improvement - Add callsign enhancement system with airline/airport lookups - Update Debian packaging with database directory and permissions Database features: - Aircraft position history with configurable retention - External aviation data loading (airlines, airports) - Callsign parsing and enhancement - API client for external lookups (OpenSky, etc.) - Privacy mode for complete offline operation CLI commands: - skyview-data status: Show database statistics - skyview-data update: Load aviation reference data - skyview-data list: Show available data sources - skyview-data clear: Remove specific data sources 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
cd51d3ecc0
commit
37c4fa2b57
25 changed files with 4771 additions and 12 deletions
411
internal/database/manager_history.go
Normal file
411
internal/database/manager_history.go
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HistoryManager struct {
|
||||
db *sql.DB
|
||||
mutex sync.RWMutex
|
||||
|
||||
// Configuration
|
||||
maxHistoryDays int
|
||||
cleanupTicker *time.Ticker
|
||||
stopCleanup chan bool
|
||||
}
|
||||
|
||||
func NewHistoryManager(db *sql.DB, maxHistoryDays int) *HistoryManager {
|
||||
hm := &HistoryManager{
|
||||
db: db,
|
||||
maxHistoryDays: maxHistoryDays,
|
||||
stopCleanup: make(chan bool),
|
||||
}
|
||||
|
||||
// Start periodic cleanup (every hour)
|
||||
hm.cleanupTicker = time.NewTicker(1 * time.Hour)
|
||||
go hm.periodicCleanup()
|
||||
|
||||
return hm
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) Close() {
|
||||
if hm.cleanupTicker != nil {
|
||||
hm.cleanupTicker.Stop()
|
||||
}
|
||||
if hm.stopCleanup != nil {
|
||||
close(hm.stopCleanup)
|
||||
}
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) periodicCleanup() {
|
||||
for {
|
||||
select {
|
||||
case <-hm.cleanupTicker.C:
|
||||
if err := hm.CleanupOldHistory(); err != nil {
|
||||
fmt.Printf("Warning: failed to cleanup old history: %v\n", err)
|
||||
}
|
||||
case <-hm.stopCleanup:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) RecordAircraft(record *AircraftHistoryRecord) error {
|
||||
hm.mutex.Lock()
|
||||
defer hm.mutex.Unlock()
|
||||
|
||||
query := `
|
||||
INSERT INTO aircraft_history
|
||||
(icao, callsign, squawk, latitude, longitude, altitude,
|
||||
vertical_rate, speed, track, source_id, signal_strength, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
_, err := hm.db.Exec(query,
|
||||
record.ICAO,
|
||||
record.Callsign,
|
||||
record.Squawk,
|
||||
record.Latitude,
|
||||
record.Longitude,
|
||||
record.Altitude,
|
||||
record.VerticalRate,
|
||||
record.Speed,
|
||||
record.Track,
|
||||
record.SourceID,
|
||||
record.SignalStrength,
|
||||
record.Timestamp,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) RecordAircraftBatch(records []AircraftHistoryRecord) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
hm.mutex.Lock()
|
||||
defer hm.mutex.Unlock()
|
||||
|
||||
tx, err := hm.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
INSERT INTO aircraft_history
|
||||
(icao, callsign, squawk, latitude, longitude, altitude,
|
||||
vertical_rate, speed, track, source_id, signal_strength, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, record := range records {
|
||||
_, err := stmt.Exec(
|
||||
record.ICAO,
|
||||
record.Callsign,
|
||||
record.Squawk,
|
||||
record.Latitude,
|
||||
record.Longitude,
|
||||
record.Altitude,
|
||||
record.VerticalRate,
|
||||
record.Speed,
|
||||
record.Track,
|
||||
record.SourceID,
|
||||
record.SignalStrength,
|
||||
record.Timestamp,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert record for ICAO %s: %w", record.ICAO, err)
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetAircraftHistory(icao string, hours int) ([]AircraftHistoryRecord, error) {
|
||||
hm.mutex.RLock()
|
||||
defer hm.mutex.RUnlock()
|
||||
|
||||
since := time.Now().Add(-time.Duration(hours) * time.Hour)
|
||||
|
||||
query := `
|
||||
SELECT icao, callsign, squawk, latitude, longitude, altitude,
|
||||
vertical_rate, speed, track, source_id, signal_strength, timestamp
|
||||
FROM aircraft_history
|
||||
WHERE icao = ? AND timestamp >= ?
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1000
|
||||
`
|
||||
|
||||
rows, err := hm.db.Query(query, icao, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var records []AircraftHistoryRecord
|
||||
for rows.Next() {
|
||||
var record AircraftHistoryRecord
|
||||
err := rows.Scan(
|
||||
&record.ICAO,
|
||||
&record.Callsign,
|
||||
&record.Squawk,
|
||||
&record.Latitude,
|
||||
&record.Longitude,
|
||||
&record.Altitude,
|
||||
&record.VerticalRate,
|
||||
&record.Speed,
|
||||
&record.Track,
|
||||
&record.SourceID,
|
||||
&record.SignalStrength,
|
||||
&record.Timestamp,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
|
||||
return records, rows.Err()
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetAircraftTrack(icao string, hours int) ([]TrackPoint, error) {
|
||||
hm.mutex.RLock()
|
||||
defer hm.mutex.RUnlock()
|
||||
|
||||
since := time.Now().Add(-time.Duration(hours) * time.Hour)
|
||||
|
||||
query := `
|
||||
SELECT latitude, longitude, altitude, timestamp
|
||||
FROM aircraft_history
|
||||
WHERE icao = ? AND timestamp >= ?
|
||||
AND latitude IS NOT NULL AND longitude IS NOT NULL
|
||||
ORDER BY timestamp ASC
|
||||
LIMIT 500
|
||||
`
|
||||
|
||||
rows, err := hm.db.Query(query, icao, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var track []TrackPoint
|
||||
for rows.Next() {
|
||||
var point TrackPoint
|
||||
err := rows.Scan(
|
||||
&point.Latitude,
|
||||
&point.Longitude,
|
||||
&point.Altitude,
|
||||
&point.Timestamp,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
track = append(track, point)
|
||||
}
|
||||
|
||||
return track, rows.Err()
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetRecentAircraft(hours int, limit int) ([]string, error) {
|
||||
hm.mutex.RLock()
|
||||
defer hm.mutex.RUnlock()
|
||||
|
||||
since := time.Now().Add(-time.Duration(hours) * time.Hour)
|
||||
|
||||
query := `
|
||||
SELECT DISTINCT icao
|
||||
FROM aircraft_history
|
||||
WHERE timestamp >= ?
|
||||
ORDER BY MAX(timestamp) DESC
|
||||
LIMIT ?
|
||||
`
|
||||
|
||||
rows, err := hm.db.Query(query, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var icaos []string
|
||||
for rows.Next() {
|
||||
var icao string
|
||||
err := rows.Scan(&icao)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
icaos = append(icaos, icao)
|
||||
}
|
||||
|
||||
return icaos, rows.Err()
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetAircraftLastSeen(icao string) (time.Time, error) {
|
||||
hm.mutex.RLock()
|
||||
defer hm.mutex.RUnlock()
|
||||
|
||||
query := `
|
||||
SELECT MAX(timestamp)
|
||||
FROM aircraft_history
|
||||
WHERE icao = ?
|
||||
`
|
||||
|
||||
var lastSeen time.Time
|
||||
err := hm.db.QueryRow(query, icao).Scan(&lastSeen)
|
||||
return lastSeen, err
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) CleanupOldHistory() error {
|
||||
hm.mutex.Lock()
|
||||
defer hm.mutex.Unlock()
|
||||
|
||||
if hm.maxHistoryDays <= 0 {
|
||||
return nil // No cleanup if maxHistoryDays is 0 or negative
|
||||
}
|
||||
|
||||
cutoff := time.Now().AddDate(0, 0, -hm.maxHistoryDays)
|
||||
|
||||
query := `DELETE FROM aircraft_history WHERE timestamp < ?`
|
||||
result, err := hm.db.Exec(query, cutoff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err == nil && rowsAffected > 0 {
|
||||
fmt.Printf("Cleaned up %d old aircraft history records\n", rowsAffected)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetStatistics() (map[string]interface{}, error) {
|
||||
return hm.GetHistoryStats()
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetHistoryStats() (map[string]interface{}, error) {
|
||||
hm.mutex.RLock()
|
||||
defer hm.mutex.RUnlock()
|
||||
|
||||
stats := make(map[string]interface{})
|
||||
|
||||
// Total records
|
||||
var totalRecords int
|
||||
err := hm.db.QueryRow(`SELECT COUNT(*) FROM aircraft_history`).Scan(&totalRecords)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats["total_records"] = totalRecords
|
||||
|
||||
// Unique aircraft
|
||||
var uniqueAircraft int
|
||||
err = hm.db.QueryRow(`SELECT COUNT(DISTINCT icao) FROM aircraft_history`).Scan(&uniqueAircraft)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats["unique_aircraft"] = uniqueAircraft
|
||||
|
||||
// Recent records (last 24 hours)
|
||||
var recentRecords int
|
||||
since := time.Now().Add(-24 * time.Hour)
|
||||
err = hm.db.QueryRow(`SELECT COUNT(*) FROM aircraft_history WHERE timestamp >= ?`, since).Scan(&recentRecords)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stats["recent_records_24h"] = recentRecords
|
||||
|
||||
// Oldest and newest record timestamps (only if records exist)
|
||||
if totalRecords > 0 {
|
||||
var oldestTimestamp, newestTimestamp time.Time
|
||||
err = hm.db.QueryRow(`SELECT MIN(timestamp), MAX(timestamp) FROM aircraft_history`).Scan(&oldestTimestamp, &newestTimestamp)
|
||||
if err == nil {
|
||||
stats["oldest_record"] = oldestTimestamp
|
||||
stats["newest_record"] = newestTimestamp
|
||||
stats["history_days"] = int(time.Since(oldestTimestamp).Hours() / 24)
|
||||
}
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (hm *HistoryManager) GetActivitySummary(hours int) (map[string]interface{}, error) {
|
||||
hm.mutex.RLock()
|
||||
defer hm.mutex.RUnlock()
|
||||
|
||||
since := time.Now().Add(-time.Duration(hours) * time.Hour)
|
||||
|
||||
summary := make(map[string]interface{})
|
||||
|
||||
// Aircraft count in time period
|
||||
var aircraftCount int
|
||||
err := hm.db.QueryRow(`
|
||||
SELECT COUNT(DISTINCT icao)
|
||||
FROM aircraft_history
|
||||
WHERE timestamp >= ?
|
||||
`, since).Scan(&aircraftCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
summary["aircraft_count"] = aircraftCount
|
||||
|
||||
// Message count in time period
|
||||
var messageCount int
|
||||
err = hm.db.QueryRow(`
|
||||
SELECT COUNT(*)
|
||||
FROM aircraft_history
|
||||
WHERE timestamp >= ?
|
||||
`, since).Scan(&messageCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
summary["message_count"] = messageCount
|
||||
|
||||
// Most active sources
|
||||
query := `
|
||||
SELECT source_id, COUNT(*) as count
|
||||
FROM aircraft_history
|
||||
WHERE timestamp >= ?
|
||||
GROUP BY source_id
|
||||
ORDER BY count DESC
|
||||
LIMIT 5
|
||||
`
|
||||
|
||||
rows, err := hm.db.Query(query, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
sources := make([]map[string]interface{}, 0)
|
||||
for rows.Next() {
|
||||
var sourceID string
|
||||
var count int
|
||||
err := rows.Scan(&sourceID, &count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sources = append(sources, map[string]interface{}{
|
||||
"source_id": sourceID,
|
||||
"count": count,
|
||||
})
|
||||
}
|
||||
summary["top_sources"] = sources
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
type TrackPoint struct {
|
||||
Latitude float64 `json:"latitude"`
|
||||
Longitude float64 `json:"longitude"`
|
||||
Altitude *int `json:"altitude,omitempty"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue