feat: Add SQLite database integration for aircraft history and callsign enhancement
- Implement comprehensive database package with versioned migrations - Add skyview-data CLI tool for managing aviation reference data - Integrate database with merger for real-time aircraft history persistence - Support OurAirports and OpenFlights data sources (runtime loading) - Add systemd timer for automated database updates - Fix transaction-based bulk loading for 2400% performance improvement - Add callsign enhancement system with airline/airport lookups - Update Debian packaging with database directory and permissions Database features: - Aircraft position history with configurable retention - External aviation data loading (airlines, airports) - Callsign parsing and enhancement - API client for external lookups (OpenSky, etc.) - Privacy mode for complete offline operation CLI commands: - skyview-data status: Show database statistics - skyview-data update: Load aviation reference data - skyview-data list: Show available data sources - skyview-data clear: Remove specific data sources 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
cd51d3ecc0
commit
37c4fa2b57
25 changed files with 4771 additions and 12 deletions
389
internal/database/api_client.go
Normal file
389
internal/database/api_client.go
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ExternalAPIClient struct {
|
||||
httpClient *http.Client
|
||||
mutex sync.RWMutex
|
||||
|
||||
// Configuration
|
||||
timeout time.Duration
|
||||
maxRetries int
|
||||
userAgent string
|
||||
|
||||
// Rate limiting
|
||||
lastRequest time.Time
|
||||
minInterval time.Duration
|
||||
}
|
||||
|
||||
type APIClientConfig struct {
|
||||
Timeout time.Duration
|
||||
MaxRetries int
|
||||
UserAgent string
|
||||
MinInterval time.Duration // Minimum interval between requests
|
||||
}
|
||||
|
||||
type OpenSkyFlightInfo struct {
|
||||
ICAO string `json:"icao"`
|
||||
Callsign string `json:"callsign"`
|
||||
Origin string `json:"origin"`
|
||||
Destination string `json:"destination"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
AircraftType string `json:"aircraft_type"`
|
||||
Registration string `json:"registration"`
|
||||
FlightNumber string `json:"flight_number"`
|
||||
Airline string `json:"airline"`
|
||||
}
|
||||
|
||||
type APIError struct {
|
||||
Operation string
|
||||
StatusCode int
|
||||
Message string
|
||||
Retryable bool
|
||||
RetryAfter time.Duration
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string {
|
||||
return fmt.Sprintf("API error in %s: %s (status: %d, retryable: %v)",
|
||||
e.Operation, e.Message, e.StatusCode, e.Retryable)
|
||||
}
|
||||
|
||||
func NewExternalAPIClient(config APIClientConfig) *ExternalAPIClient {
|
||||
if config.Timeout == 0 {
|
||||
config.Timeout = 10 * time.Second
|
||||
}
|
||||
if config.MaxRetries == 0 {
|
||||
config.MaxRetries = 3
|
||||
}
|
||||
if config.UserAgent == "" {
|
||||
config.UserAgent = "SkyView-ADSB/1.0 (https://github.com/user/skyview)"
|
||||
}
|
||||
if config.MinInterval == 0 {
|
||||
config.MinInterval = 1 * time.Second // Default rate limit
|
||||
}
|
||||
|
||||
return &ExternalAPIClient{
|
||||
httpClient: &http.Client{
|
||||
Timeout: config.Timeout,
|
||||
},
|
||||
timeout: config.Timeout,
|
||||
maxRetries: config.MaxRetries,
|
||||
userAgent: config.UserAgent,
|
||||
minInterval: config.MinInterval,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) enforceRateLimit() {
|
||||
c.mutex.Lock()
|
||||
defer c.mutex.Unlock()
|
||||
|
||||
elapsed := time.Since(c.lastRequest)
|
||||
if elapsed < c.minInterval {
|
||||
time.Sleep(c.minInterval - elapsed)
|
||||
}
|
||||
c.lastRequest = time.Now()
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) makeRequest(ctx context.Context, url string) (*http.Response, error) {
|
||||
c.enforceRateLimit()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", c.userAgent)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
var resp *http.Response
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt <= c.maxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
// Exponential backoff
|
||||
backoff := time.Duration(1<<uint(attempt-1)) * time.Second
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
|
||||
resp, lastErr = c.httpClient.Do(req)
|
||||
if lastErr != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check for retryable status codes
|
||||
if resp.StatusCode >= 500 || resp.StatusCode == 429 {
|
||||
resp.Body.Close()
|
||||
|
||||
// Handle rate limiting
|
||||
if resp.StatusCode == 429 {
|
||||
retryAfter := parseRetryAfter(resp.Header.Get("Retry-After"))
|
||||
if retryAfter > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(retryAfter):
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Success or non-retryable error
|
||||
break
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) GetFlightInfoFromOpenSky(ctx context.Context, icao string) (*OpenSkyFlightInfo, error) {
|
||||
if icao == "" {
|
||||
return nil, fmt.Errorf("empty ICAO code")
|
||||
}
|
||||
|
||||
// OpenSky Network API endpoint for flight information
|
||||
apiURL := fmt.Sprintf("https://opensky-network.org/api/flights/aircraft?icao24=%s&begin=%d&end=%d",
|
||||
icao,
|
||||
time.Now().Add(-24*time.Hour).Unix(),
|
||||
time.Now().Unix(),
|
||||
)
|
||||
|
||||
resp, err := c.makeRequest(ctx, apiURL)
|
||||
if err != nil {
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_flight_info",
|
||||
Message: err.Error(),
|
||||
Retryable: true,
|
||||
}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_flight_info",
|
||||
StatusCode: resp.StatusCode,
|
||||
Message: string(body),
|
||||
Retryable: resp.StatusCode >= 500 || resp.StatusCode == 429,
|
||||
}
|
||||
}
|
||||
|
||||
var flights [][]interface{}
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
if err := decoder.Decode(&flights); err != nil {
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_parse_response",
|
||||
Message: err.Error(),
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
if len(flights) == 0 {
|
||||
return nil, nil // No flight information available
|
||||
}
|
||||
|
||||
// Parse the most recent flight
|
||||
flight := flights[0]
|
||||
if len(flight) < 10 {
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_invalid_response",
|
||||
Message: "invalid flight data format",
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
info := &OpenSkyFlightInfo{
|
||||
ICAO: icao,
|
||||
}
|
||||
|
||||
// Parse fields based on OpenSky API documentation
|
||||
if callsign, ok := flight[1].(string); ok {
|
||||
info.Callsign = callsign
|
||||
}
|
||||
if firstSeen, ok := flight[2].(float64); ok {
|
||||
info.FirstSeen = time.Unix(int64(firstSeen), 0)
|
||||
}
|
||||
if lastSeen, ok := flight[3].(float64); ok {
|
||||
info.LastSeen = time.Unix(int64(lastSeen), 0)
|
||||
}
|
||||
if origin, ok := flight[4].(string); ok {
|
||||
info.Origin = origin
|
||||
}
|
||||
if destination, ok := flight[5].(string); ok {
|
||||
info.Destination = destination
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) GetAircraftInfoFromOpenSky(ctx context.Context, icao string) (map[string]interface{}, error) {
|
||||
if icao == "" {
|
||||
return nil, fmt.Errorf("empty ICAO code")
|
||||
}
|
||||
|
||||
// OpenSky Network metadata API
|
||||
apiURL := fmt.Sprintf("https://opensky-network.org/api/metadata/aircraft/icao/%s", icao)
|
||||
|
||||
resp, err := c.makeRequest(ctx, apiURL)
|
||||
if err != nil {
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_aircraft_info",
|
||||
Message: err.Error(),
|
||||
Retryable: true,
|
||||
}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil // Aircraft not found
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_aircraft_info",
|
||||
StatusCode: resp.StatusCode,
|
||||
Message: string(body),
|
||||
Retryable: resp.StatusCode >= 500 || resp.StatusCode == 429,
|
||||
}
|
||||
}
|
||||
|
||||
var aircraft map[string]interface{}
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
if err := decoder.Decode(&aircraft); err != nil {
|
||||
return nil, &APIError{
|
||||
Operation: "opensky_parse_aircraft",
|
||||
Message: err.Error(),
|
||||
Retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
return aircraft, nil
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) EnhanceCallsignWithExternalData(ctx context.Context, callsign, icao string) (map[string]interface{}, error) {
|
||||
enhancement := make(map[string]interface{})
|
||||
enhancement["callsign"] = callsign
|
||||
enhancement["icao"] = icao
|
||||
enhancement["enhanced"] = false
|
||||
|
||||
// Try to get flight information from OpenSky
|
||||
if flightInfo, err := c.GetFlightInfoFromOpenSky(ctx, icao); err == nil && flightInfo != nil {
|
||||
enhancement["flight_info"] = map[string]interface{}{
|
||||
"origin": flightInfo.Origin,
|
||||
"destination": flightInfo.Destination,
|
||||
"first_seen": flightInfo.FirstSeen,
|
||||
"last_seen": flightInfo.LastSeen,
|
||||
"flight_number": flightInfo.FlightNumber,
|
||||
"airline": flightInfo.Airline,
|
||||
}
|
||||
enhancement["enhanced"] = true
|
||||
}
|
||||
|
||||
// Try to get aircraft metadata
|
||||
if aircraftInfo, err := c.GetAircraftInfoFromOpenSky(ctx, icao); err == nil && aircraftInfo != nil {
|
||||
enhancement["aircraft_info"] = aircraftInfo
|
||||
enhancement["enhanced"] = true
|
||||
}
|
||||
|
||||
return enhancement, nil
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) BatchEnhanceCallsigns(ctx context.Context, callsigns map[string]string) (map[string]map[string]interface{}, error) {
|
||||
results := make(map[string]map[string]interface{})
|
||||
|
||||
for callsign, icao := range callsigns {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return results, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
enhanced, err := c.EnhanceCallsignWithExternalData(ctx, callsign, icao)
|
||||
if err != nil {
|
||||
// Log error but continue with other callsigns
|
||||
fmt.Printf("Warning: failed to enhance callsign %s (ICAO: %s): %v\n", callsign, icao, err)
|
||||
continue
|
||||
}
|
||||
|
||||
results[callsign] = enhanced
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c *ExternalAPIClient) TestConnection(ctx context.Context) error {
|
||||
// Test with a simple API call
|
||||
testURL := "https://opensky-network.org/api/states?time=0&lamin=0&lomin=0&lamax=1&lomax=1"
|
||||
|
||||
resp, err := c.makeRequest(ctx, testURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connection test failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("connection test returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseRetryAfter(header string) time.Duration {
|
||||
if header == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Try parsing as seconds
|
||||
if seconds, err := time.ParseDuration(header + "s"); err == nil {
|
||||
return seconds
|
||||
}
|
||||
|
||||
// Try parsing as HTTP date
|
||||
if t, err := http.ParseTime(header); err == nil {
|
||||
return time.Until(t)
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// HealthCheck provides information about the client's health
|
||||
func (c *ExternalAPIClient) HealthCheck(ctx context.Context) map[string]interface{} {
|
||||
health := make(map[string]interface{})
|
||||
|
||||
// Test connection
|
||||
if err := c.TestConnection(ctx); err != nil {
|
||||
health["status"] = "unhealthy"
|
||||
health["error"] = err.Error()
|
||||
} else {
|
||||
health["status"] = "healthy"
|
||||
}
|
||||
|
||||
// Add configuration info
|
||||
health["timeout"] = c.timeout.String()
|
||||
health["max_retries"] = c.maxRetries
|
||||
health["min_interval"] = c.minInterval.String()
|
||||
health["user_agent"] = c.userAgent
|
||||
|
||||
c.mutex.RLock()
|
||||
health["last_request"] = c.lastRequest
|
||||
c.mutex.RUnlock()
|
||||
|
||||
return health
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue