diff --git a/assets/assets.go b/assets/assets.go index e80edaa..54e1c4a 100644 --- a/assets/assets.go +++ b/assets/assets.go @@ -28,6 +28,5 @@ import "embed" // This approach ensures the web interface is always available without requiring // external file deployment or complicated asset management. // -// Updated to include database.html for database status page //go:embed static var Static embed.FS diff --git a/assets/static/css/style.css b/assets/static/css/style.css index 3879d96..5389c2d 100644 --- a/assets/static/css/style.css +++ b/assets/static/css/style.css @@ -566,95 +566,6 @@ body { color: #00ff88 !important; } -/* Rich callsign display styles */ -.callsign-display { - display: inline-block; -} - -.callsign-display.enriched { - display: inline-flex; - flex-direction: column; - gap: 0.25rem; -} - -.callsign-code { - display: inline-flex; - align-items: center; - gap: 0.25rem; -} - -.airline-code { - color: #00ff88 !important; - font-weight: 600; - font-family: monospace; - background: rgba(0, 255, 136, 0.1); - padding: 0.1rem 0.3rem; - border-radius: 3px; - border: 1px solid rgba(0, 255, 136, 0.3); -} - -.flight-number { - color: #00a8ff !important; - font-weight: 500; - font-family: monospace; -} - -.callsign-details { - font-size: 0.85rem; - opacity: 0.9; -} - -.airline-name { - color: #ffd700 !important; - font-weight: 500; -} - -.airline-country { - color: #cccccc !important; - font-size: 0.8rem; - opacity: 0.8; -} - -.callsign-display.simple { - color: #00ff88 !important; - font-family: monospace; -} - -.callsign-display.no-data { - color: #888888 !important; - font-style: italic; -} - -/* Compact callsign for table view */ -.callsign-compact { - color: #00ff88 !important; - font-family: monospace; - font-weight: 500; -} - -/* Loading state for callsign enhancement */ -.callsign-loading { - position: relative; -} - -.callsign-loading::after { - content: '⟳'; - margin-left: 0.25rem; - opacity: 0.6; - animation: spin 1s linear infinite; - font-size: 0.8rem; -} - -@keyframes spin { - from { transform: rotate(0deg); } - to { transform: rotate(360deg); } -} - -.callsign-enhanced { - /* Smooth transition when enhanced */ - transition: all 0.3s ease; -} - .popup-details { font-size: 0.9rem; color: #ffffff !important; diff --git a/assets/static/index.html b/assets/static/index.html index 2994330..5256e82 100644 --- a/assets/static/index.html +++ b/assets/static/index.html @@ -28,10 +28,7 @@
-

SkyView v0.0.8 - ⚙ - 📊 -

+

SkyView v0.0.8 ⚙

diff --git a/assets/static/js/app.js b/assets/static/js/app.js index 4ba39d3..59b7dae 100644 --- a/assets/static/js/app.js +++ b/assets/static/js/app.js @@ -7,7 +7,6 @@ import { WebSocketManager } from './modules/websocket.js?v=2'; import { AircraftManager } from './modules/aircraft-manager.js?v=2'; import { MapManager } from './modules/map-manager.js?v=2'; import { UIManager } from './modules/ui-manager.js?v=2'; -import { CallsignManager } from './modules/callsign-manager.js'; class SkyView { constructor() { @@ -16,7 +15,6 @@ class SkyView { this.aircraftManager = null; this.mapManager = null; this.uiManager = null; - this.callsignManager = null; // 3D Radar this.radar3d = null; @@ -39,15 +37,12 @@ class SkyView { this.uiManager.initializeViews(); this.uiManager.initializeEventListeners(); - // Initialize callsign manager for enriched callsign display - this.callsignManager = new CallsignManager(); - // Initialize map manager and get the main map this.mapManager = new MapManager(); const map = await this.mapManager.initializeMap(); - // Initialize aircraft manager with the map and callsign manager - this.aircraftManager = new AircraftManager(map, this.callsignManager); + // Initialize aircraft manager with the map + this.aircraftManager = new AircraftManager(map); // Set up selected aircraft trail callback this.aircraftManager.setSelectedAircraftCallback((icao) => { diff --git a/assets/static/js/modules/aircraft-manager.js b/assets/static/js/modules/aircraft-manager.js index 78f5ada..8346bc9 100644 --- a/assets/static/js/modules/aircraft-manager.js +++ b/assets/static/js/modules/aircraft-manager.js @@ -1,8 +1,7 @@ // Aircraft marker and data management module export class AircraftManager { - constructor(map, callsignManager = null) { + constructor(map) { this.map = map; - this.callsignManager = callsignManager; this.aircraftData = new Map(); this.aircraftMarkers = new Map(); this.aircraftTrails = new Map(); @@ -229,11 +228,6 @@ export class AircraftManager { // Handle popup exactly like Leaflet expects if (marker.isPopupOpen()) { marker.setPopupContent(this.createPopupContent(aircraft)); - // Enhance callsign display for updated popup - const popupElement = marker.getPopup().getElement(); - if (popupElement) { - this.enhanceCallsignDisplay(popupElement); - } } this.markerUpdateCount++; @@ -256,14 +250,6 @@ export class AircraftManager { maxWidth: 450, className: 'aircraft-popup' }); - - // Enhance callsign display when popup opens - marker.on('popupopen', (e) => { - const popupElement = e.popup.getElement(); - if (popupElement) { - this.enhanceCallsignDisplay(popupElement); - } - }); this.aircraftMarkers.set(icao, marker); this.markerCreateCount++; @@ -449,7 +435,7 @@ export class AircraftManager {
${flag} ${aircraft.ICAO24 || 'N/A'} - ${aircraft.Callsign ? `→ ${aircraft.Callsign}` : ''} + ${aircraft.Callsign ? `→ ${aircraft.Callsign}` : ''}
@@ -525,29 +511,6 @@ export class AircraftManager { return minDistance === Infinity ? null : minDistance; } - // Enhance callsign display in popup after it's created - async enhanceCallsignDisplay(popupElement) { - if (!this.callsignManager) return; - - const callsignElements = popupElement.querySelectorAll('.callsign-loading'); - - for (const element of callsignElements) { - const callsign = element.dataset.callsign; - if (!callsign) continue; - - try { - const callsignInfo = await this.callsignManager.getCallsignInfo(callsign); - const richDisplay = this.callsignManager.generateCallsignDisplay(callsignInfo, callsign); - element.innerHTML = richDisplay; - element.classList.remove('callsign-loading'); - element.classList.add('callsign-enhanced'); - } catch (error) { - console.warn(`Failed to enhance callsign display for ${callsign}:`, error); - // Keep the simple display on error - element.classList.remove('callsign-loading'); - } - } - } toggleTrails() { this.showTrails = !this.showTrails; diff --git a/assets/static/js/modules/callsign-manager.js b/assets/static/js/modules/callsign-manager.js deleted file mode 100644 index ef9a089..0000000 --- a/assets/static/js/modules/callsign-manager.js +++ /dev/null @@ -1,163 +0,0 @@ -// Callsign enrichment and display module -export class CallsignManager { - constructor() { - this.callsignCache = new Map(); - this.pendingRequests = new Map(); - - // Rate limiting to avoid overwhelming the API - this.lastRequestTime = 0; - this.requestInterval = 100; // Minimum 100ms between requests - } - - /** - * Get enriched callsign information, using cache when available - * @param {string} callsign - The raw callsign to lookup - * @returns {Promise} - Enriched callsign data - */ - async getCallsignInfo(callsign) { - if (!callsign || callsign.trim() === '') { - return null; - } - - const cleanCallsign = callsign.trim().toUpperCase(); - - // Check cache first - if (this.callsignCache.has(cleanCallsign)) { - return this.callsignCache.get(cleanCallsign); - } - - // Check if we already have a pending request for this callsign - if (this.pendingRequests.has(cleanCallsign)) { - return this.pendingRequests.get(cleanCallsign); - } - - // Rate limiting - const now = Date.now(); - if (now - this.lastRequestTime < this.requestInterval) { - await new Promise(resolve => setTimeout(resolve, this.requestInterval)); - } - - // Create the API request - const requestPromise = this.fetchCallsignInfo(cleanCallsign); - this.pendingRequests.set(cleanCallsign, requestPromise); - - try { - const result = await requestPromise; - - // Cache the result for future use - if (result && result.callsign) { - this.callsignCache.set(cleanCallsign, result.callsign); - } - - return result ? result.callsign : null; - } catch (error) { - console.warn(`Failed to lookup callsign ${cleanCallsign}:`, error); - return null; - } finally { - // Clean up pending request - this.pendingRequests.delete(cleanCallsign); - this.lastRequestTime = Date.now(); - } - } - - /** - * Fetch callsign information from the API - * @param {string} callsign - The callsign to lookup - * @returns {Promise} - API response - */ - async fetchCallsignInfo(callsign) { - const response = await fetch(`/api/callsign/${encodeURIComponent(callsign)}`); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - return await response.json(); - } - - /** - * Generate rich HTML display for a callsign - * @param {Object} callsignInfo - Enriched callsign data from API - * @param {string} originalCallsign - Original callsign if API data is null - * @returns {string} - HTML string for display - */ - generateCallsignDisplay(callsignInfo, originalCallsign = '') { - if (!callsignInfo || !callsignInfo.is_valid) { - // Fallback for invalid or missing callsign data - if (originalCallsign) { - return `${originalCallsign}`; - } - return 'N/A'; - } - - const parts = []; - - // Airline code - if (callsignInfo.airline_code) { - parts.push(`${callsignInfo.airline_code}`); - } - - // Flight number - if (callsignInfo.flight_number) { - parts.push(`${callsignInfo.flight_number}`); - } - - // Airline name (if available) - let airlineInfo = ''; - if (callsignInfo.airline_name) { - airlineInfo = ` - ${callsignInfo.airline_name} - `; - - // Add country if available - if (callsignInfo.airline_country) { - airlineInfo += ` (${callsignInfo.airline_country})`; - } - } - - return ` - - ${parts.join(' ')} - ${airlineInfo ? `${airlineInfo}` : ''} - - `; - } - - /** - * Generate compact callsign display for table view - * @param {Object} callsignInfo - Enriched callsign data - * @param {string} originalCallsign - Original callsign fallback - * @returns {string} - Compact HTML for table display - */ - generateCompactCallsignDisplay(callsignInfo, originalCallsign = '') { - if (!callsignInfo || !callsignInfo.is_valid) { - return originalCallsign || 'N/A'; - } - - // For tables, use the display_name or format airline + flight - if (callsignInfo.display_name) { - return `${callsignInfo.display_name}`; - } - - return `${callsignInfo.airline_code} ${callsignInfo.flight_number}`; - } - - /** - * Clear the callsign cache (useful for memory management) - */ - clearCache() { - this.callsignCache.clear(); - console.debug('Callsign cache cleared'); - } - - /** - * Get cache statistics for debugging - * @returns {Object} - Cache size and pending requests - */ - getCacheStats() { - return { - cacheSize: this.callsignCache.size, - pendingRequests: this.pendingRequests.size - }; - } -} \ No newline at end of file diff --git a/debian/usr/share/doc/skyview-adsb/DATABASE.md b/debian/usr/share/doc/skyview-adsb/DATABASE.md index 326f1cf..2e7347d 100644 --- a/debian/usr/share/doc/skyview-adsb/DATABASE.md +++ b/debian/usr/share/doc/skyview-adsb/DATABASE.md @@ -1,729 +1,99 @@ -# SkyView Database Architecture +# SkyView Database Management -This document describes SkyView's SQLite database architecture, migration system, and integration approach for persistent data storage. +SkyView includes a comprehensive database management system for enriching aircraft callsigns with airline and airport information. -## Overview +## Quick Start -SkyView uses a single SQLite database to store: -- **Historic aircraft data**: Position history, message counts, signal strength -- **Callsign lookup data**: Cached airline/airport information from external APIs -- **Embedded aviation data**: OpenFlights airline and airport databases - -## Database Design Principles - -### Embedded Architecture -- Single SQLite file for all persistent data -- No external database dependencies -- Self-contained deployment with embedded schemas -- Backward compatibility through versioned migrations - -### Performance Optimization -- Strategic indexing for time-series aircraft data -- Efficient lookups for callsign enhancement -- Configurable data retention policies -- Query optimization for real-time operations - -### Data Safety -- Atomic migration transactions -- Pre-migration backups for destructive changes -- Data loss warnings for schema changes -- Rollback capabilities where possible - -## Database Schema - -### Core Tables - -#### `schema_info` -Tracks database version and applied migrations: -```sql -CREATE TABLE schema_info ( - version INTEGER PRIMARY KEY, - applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - description TEXT, - checksum TEXT -); -``` - -#### `aircraft_history` -Stores time-series aircraft position and message data: -```sql -CREATE TABLE aircraft_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - icao TEXT NOT NULL, - timestamp TIMESTAMP NOT NULL, - latitude REAL, - longitude REAL, - altitude INTEGER, - speed INTEGER, - track INTEGER, - vertical_rate INTEGER, - squawk TEXT, - callsign TEXT, - source_id TEXT NOT NULL, - signal_strength REAL -); -``` - -**Indexes:** -- `idx_aircraft_history_icao_time`: Fast queries by aircraft and time range -- `idx_aircraft_history_timestamp`: Time-based cleanup and queries -- `idx_aircraft_history_callsign`: Callsign-based searches - -#### `airlines` -Multi-source airline database with unified schema: -```sql -CREATE TABLE airlines ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - alias TEXT, - iata_code TEXT, - icao_code TEXT, - callsign TEXT, - country TEXT, - country_code TEXT, - active BOOLEAN DEFAULT 1, - data_source TEXT NOT NULL DEFAULT 'unknown', - source_id TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); -``` - -**Indexes:** -- `idx_airlines_icao_code`: ICAO code lookup (primary for callsign enhancement) -- `idx_airlines_iata_code`: IATA code lookup -- `idx_airlines_callsign`: Radio callsign lookup -- `idx_airlines_country_code`: Country-based filtering -- `idx_airlines_active`: Active airlines filtering -- `idx_airlines_source`: Data source tracking - -#### `airports` -Multi-source airport database with comprehensive metadata: -```sql -CREATE TABLE airports ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - ident TEXT, - type TEXT, - city TEXT, - municipality TEXT, - region TEXT, - country TEXT, - country_code TEXT, - continent TEXT, - iata_code TEXT, - icao_code TEXT, - local_code TEXT, - gps_code TEXT, - latitude REAL, - longitude REAL, - elevation_ft INTEGER, - scheduled_service BOOLEAN DEFAULT 0, - home_link TEXT, - wikipedia_link TEXT, - keywords TEXT, - timezone_offset REAL, - timezone TEXT, - dst_type TEXT, - data_source TEXT NOT NULL DEFAULT 'unknown', - source_id TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); -``` - -**Indexes:** -- `idx_airports_icao_code`: ICAO code lookup -- `idx_airports_iata_code`: IATA code lookup -- `idx_airports_ident`: Airport identifier lookup -- `idx_airports_country_code`: Country-based filtering -- `idx_airports_type`: Airport type filtering -- `idx_airports_coords`: Geographic coordinate queries -- `idx_airports_source`: Data source tracking - -#### `callsign_cache` -Caches external API lookups and local enrichment for callsign enhancement: -```sql -CREATE TABLE callsign_cache ( - callsign TEXT PRIMARY KEY, - airline_icao TEXT, - airline_iata TEXT, - airline_name TEXT, - airline_country TEXT, - flight_number TEXT, - origin_iata TEXT, -- Departure airport IATA code - destination_iata TEXT, -- Arrival airport IATA code - aircraft_type TEXT, - route TEXT, -- Full route description - status TEXT, -- Flight status (scheduled, delayed, etc.) - source TEXT NOT NULL DEFAULT 'local', - cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - expires_at TIMESTAMP NOT NULL -); -``` - -**Route Information Fields:** -- **`origin_iata`**: IATA code of departure airport (e.g., "JFK" for New York JFK) -- **`destination_iata`**: IATA code of arrival airport (e.g., "LAX" for Los Angeles) -- **`route`**: Human-readable route description (e.g., "JFK-LAX" or "New York to Los Angeles") -- **`status`**: Current flight status when available from external APIs - -These fields enable enhanced flight tracking with origin-destination pairs and route visualization. - -**Indexes:** -- `idx_callsign_cache_expires`: Efficient cache cleanup -- `idx_callsign_cache_airline`: Airline-based queries - -#### `data_sources` -Tracks loaded external data sources and their metadata: -```sql -CREATE TABLE data_sources ( - name TEXT PRIMARY KEY, - license TEXT NOT NULL, - url TEXT, - version TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - record_count INTEGER DEFAULT 0, - user_accepted_license BOOLEAN DEFAULT 0 -); -``` - -## Database Location Strategy - -### Path Resolution Order -1. **Explicit configuration**: `database.path` in config file -2. **System service**: `/var/lib/skyview/skyview.db` -3. **User mode**: `~/.local/share/skyview/skyview.db` -4. **Fallback**: `./skyview.db` in current directory - -### Directory Permissions -- System: `root:root` with `755` permissions for `/var/lib/skyview/` -- User: User-owned directories with standard permissions -- Service: `skyview:skyview` user/group for system service - -## Migration System - -### Migration Structure -```go -type Migration struct { - Version int // Sequential version number - Description string // Human-readable description - Up string // SQL for applying migration - Down string // SQL for rollback (optional) - DataLoss bool // Warning flag for destructive changes -} -``` - -### Migration Process -1. **Version Check**: Compare current schema version with available migrations -2. **Backup**: Create automatic backup before destructive changes -3. **Transaction**: Wrap each migration in atomic transaction -4. **Validation**: Verify schema integrity after migration -5. **Logging**: Record successful migrations in `schema_info` - -### Data Loss Protection -- Migrations marked with `DataLoss: true` require explicit user consent -- Automatic backups created before destructive operations -- Warning messages displayed during upgrade process -- Rollback SQL provided where possible - -### Example Migration Sequence -```go -var migrations = []Migration{ - { - Version: 1, - Description: "Initial schema with aircraft history", - Up: createInitialSchema, - DataLoss: false, - }, - { - Version: 2, - Description: "Add OpenFlights airline and airport data", - Up: addAviationTables, - DataLoss: false, - }, - { - Version: 3, - Description: "Add callsign lookup cache", - Up: addCallsignCache, - DataLoss: false, - }, -} -``` - -## Data Sources and Loading - -SkyView supports multiple aviation data sources with automatic conflict resolution and license compliance. - -### Supported Data Sources - -#### OpenFlights Airlines Database -- **Source**: https://openflights.org/data.html -- **License**: Open Database License (ODbL) 1.0 -- **Content**: Global airline data with ICAO/IATA codes, callsigns, and country information -- **Records**: ~6,162 airlines -- **Update Method**: Runtime download (no license confirmation required) - -#### OpenFlights Airports Database -- **Source**: https://openflights.org/data.html -- **License**: Open Database License (ODbL) 1.0 -- **Content**: Global airport data with coordinates, codes, and metadata -- **Records**: ~7,698 airports -- **Update Method**: Runtime download - -#### OurAirports Database -- **Source**: https://ourairports.com/data/ -- **License**: Creative Commons Zero (CC0) 1.0 -- **Content**: Comprehensive airport database with detailed metadata -- **Records**: ~83,557 airports -- **Update Method**: Runtime download - -### Data Loading System - -#### Intelligent Conflict Resolution -The data loading system uses **INSERT OR REPLACE** upserts to handle overlapping data: - -```sql -INSERT OR REPLACE INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) -``` - -This ensures that: -- Duplicate records are automatically updated rather than causing errors -- Later data sources can override earlier ones -- Database integrity is maintained during bulk loads - -#### Loading Process -1. **Source Validation**: Verify data source accessibility and format -2. **Incremental Processing**: Process data in chunks to manage memory -3. **Error Handling**: Log and continue on individual record errors -4. **Statistics Reporting**: Track records processed, added, and errors -5. **Source Tracking**: Record metadata about each loaded source - -#### Performance Characteristics -- **OpenFlights Airlines**: ~6,162 records in ~363ms -- **OpenFlights Airports**: ~7,698 records in ~200ms -- **OurAirports**: ~83,557 records in ~980ms -- **Error Rate**: <0.1% under normal conditions - -## Configuration Integration - -### Database Configuration -```json -{ - "database": { - "path": "/var/lib/skyview-adsb/skyview.db", - "max_history_days": 7, - "backup_on_upgrade": true, - "vacuum_interval": "24h", - "page_size": 4096 - }, - "callsign": { - "enabled": true, - "cache_hours": 24, - "external_apis": true, - "privacy_mode": false - } -} -``` - -### Configuration Fields - -#### `database` -- **`path`**: Database file location (empty = auto-resolve) -- **`max_history_days`**: Retention policy for aircraft history (0 = unlimited) -- **`backup_on_upgrade`**: Create backup before schema migrations - -#### `callsign` -- **`enabled`**: Enable callsign enhancement features -- **`cache_hours`**: TTL for cached external API results -- **`privacy_mode`**: Disable all external data requests -- **`sources`**: Independent control for each data source - -### Enhanced Configuration Example -```json -{ - "callsign": { - "enabled": true, - "cache_hours": 24, - "privacy_mode": false, - "sources": { - "openflights_embedded": { - "enabled": true, - "priority": 1, - "license": "AGPL-3.0" - }, - "faa_registry": { - "enabled": false, - "priority": 2, - "update_frequency": "weekly", - "license": "public_domain" - }, - "opensky_api": { - "enabled": false, - "priority": 3, - "timeout_seconds": 5, - "max_retries": 2, - "requires_consent": true, - "license_warning": "Commercial use requires OpenSky Network consent", - "user_accepts_terms": false - }, - "custom_database": { - "enabled": false, - "priority": 4, - "path": "", - "license": "user_verified" - } - }, - "fallback_chain": ["openflights_embedded", "faa_registry", "opensky_api", "custom_database"] - } -} -``` - -#### Individual Source Configuration Options -- **`enabled`**: Enable/disable this specific source -- **`priority`**: Processing order (lower numbers = higher priority) -- **`license`**: License type for compliance tracking -- **`requires_consent`**: Whether source requires explicit user consent -- **`user_accepts_terms`**: User acknowledgment of licensing terms -- **`timeout_seconds`**: Per-source timeout configuration -- **`max_retries`**: Per-source retry limits -- **`update_frequency`**: For downloadable sources (daily/weekly/monthly) - -## Debian Package Integration - -### Package Structure -``` -/var/lib/skyview/ # Database directory -/etc/skyview/config.json # Default configuration -/usr/bin/skyview # Main application -/usr/share/skyview/ # Embedded resources -``` - -### Installation Process -1. **`postinst`**: Create directories, user accounts, permissions -2. **First Run**: Database initialization and migration on startup -3. **Upgrades**: Automatic schema migration with backup -4. **Service**: Systemd integration with proper database access - -### Service User -- User: `skyview-adsb` -- Home: `/var/lib/skyview-adsb` -- Shell: `/bin/false` (service account) -- Database: Read/write access to `/var/lib/skyview-adsb/` - -### Automatic Database Updates -The systemd service configuration includes automatic database updates on startup: - -```ini -[Service] -Type=simple -User=skyview-adsb -Group=skyview-adsb -# Update database before starting main service -ExecStartPre=/usr/bin/skyview-data -config /etc/skyview-adsb/config.json update -TimeoutStartSec=300 -ExecStart=/usr/bin/skyview -config /etc/skyview-adsb/config.json -``` - -This ensures aviation data sources are refreshed before each service start, complementing the weekly timer-based updates. - -## Data Retention and Cleanup - -### Automatic Cleanup -- **Aircraft History**: Configurable retention period (`max_history_days`) -- **Cache Expiration**: TTL-based cleanup of external API cache -- **Optimization**: Periodic VACUUM operations for storage efficiency - -### Manual Maintenance -```sql --- Clean old aircraft history (example: 7 days) -DELETE FROM aircraft_history -WHERE timestamp < datetime('now', '-7 days'); - --- Clean expired cache entries -DELETE FROM callsign_cache -WHERE expires_at < datetime('now'); - --- Optimize database storage -VACUUM; -``` - -## Database Optimization - -SkyView includes a comprehensive database optimization system that automatically manages storage efficiency and performance. - -### Optimization Features - -#### Automatic VACUUM Operations -- **Full VACUUM**: Rebuilds database to reclaim deleted space -- **Incremental VACUUM**: Gradual space reclamation with minimal performance impact -- **Scheduled Maintenance**: Configurable intervals for automatic optimization -- **Size Reporting**: Before/after statistics with space savings metrics - -#### Storage Optimization -- **Page Size Optimization**: Configurable SQLite page size for optimal performance -- **Auto-Vacuum Configuration**: Enables incremental space reclamation -- **Statistics Updates**: ANALYZE operations for query plan optimization -- **Efficiency Monitoring**: Real-time storage efficiency reporting - -### Using the Optimization System - -#### Command Line Interface +### 1. Check Current Status ```bash -# Run comprehensive database optimization -skyview-data optimize - -# Run with force flag to skip confirmation prompts -skyview-data optimize --force - -# Check current optimization statistics -skyview-data optimize --stats-only +skyview-data status ``` -#### Optimization Output Example -``` -Optimizing database for storage efficiency... -✓ Auto VACUUM: Enable incremental auto-vacuum -✓ Incremental VACUUM: Reclaim free pages incrementally -✓ Optimize: Update SQLite query planner statistics -✓ Analyze: Update table statistics for better query plans - -VACUUM completed in 1.2s: 275.3 MB → 263.1 MB (saved 12.2 MB, 4.4%) - -Database optimization completed successfully. -Storage efficiency: 96.8% (263.1 MB used of 272.4 MB allocated) +### 2. Import Safe Data (Recommended) +```bash +# Import public domain sources automatically +skyview-data update ``` -#### Configuration Options -```json -{ - "database": { - "vacuum_interval": "24h", - "page_size": 4096, - "enable_compression": true, - "compression_level": 6 - } -} +### 3. Enable Automatic Updates (Optional) +```bash +# Weekly updates on Sunday at 3 AM +sudo systemctl enable --now skyview-database-update.timer ``` -### Optimization Statistics +## Available Data Sources -The optimization system provides detailed metrics about database performance: +### Safe Sources (Public Domain) +These sources are imported automatically with `skyview-data update`: +- **OurAirports**: Comprehensive airport database (public domain) +- **FAA Registry**: US aircraft registration data (public domain) -#### Available Statistics -- **Database Size**: Total file size in bytes -- **Page Statistics**: Page size, count, and utilization -- **Storage Efficiency**: Percentage of allocated space actually used -- **Free Space**: Amount of reclaimable space available -- **Auto-Vacuum Status**: Current auto-vacuum configuration -- **Last Optimization**: Timestamp of most recent optimization +### License-Required Sources +These require explicit acceptance: +- **OpenFlights**: Airline and airport data (AGPL-3.0 license) -#### Programmatic Access -```go -// Get current optimization statistics -optimizer := NewOptimizationManager(db, config) -stats, err := optimizer.GetOptimizationStats() -if err != nil { - log.Fatal("Failed to get stats:", err) -} +## Commands -fmt.Printf("Database efficiency: %.1f%%\n", stats.Efficiency) -fmt.Printf("Storage used: %.1f MB\n", float64(stats.DatabaseSize)/(1024*1024)) +### Basic Operations +```bash +skyview-data list # Show available sources +skyview-data status # Show database status +skyview-data update # Update safe sources +skyview-data import openflights # Import licensed source +skyview-data clear # Remove source data ``` -## Performance Considerations +### Systemd Timer Management +```bash +# Enable weekly automatic updates +systemctl enable skyview-database-update.timer +systemctl start skyview-database-update.timer -### Query Optimization -- Time-range queries use `idx_aircraft_history_icao_time` -- Callsign lookups prioritize local cache over external APIs -- Bulk operations use transactions for consistency +# Check timer status +systemctl status skyview-database-update.timer -### Storage Efficiency -- Configurable history limits prevent unbounded growth -- Automatic VACUUM operations with optimization reporting -- Compressed timestamps and efficient data types -- Page size optimization for storage efficiency -- Auto-vacuum configuration for incremental space reclamation +# View update logs +journalctl -u skyview-database-update.service -### Memory Usage -- WAL mode for concurrent read/write access -- Connection pooling for multiple goroutines -- Prepared statements for repeated queries - -## Privacy and Security - -### Privacy Mode -SkyView includes comprehensive privacy controls through the `privacy_mode` configuration option: - -```json -{ - "callsign": { - "enabled": true, - "privacy_mode": true, - "external_apis": false - } -} +# Disable automatic updates +systemctl disable skyview-database-update.timer ``` -#### Privacy Mode Features -- **No External Calls**: Completely disables all external API requests -- **Local-Only Lookups**: Uses only embedded OpenFlights database for callsign enhancement -- **No Data Transmission**: Aircraft data never leaves the local system -- **Compliance**: Suitable for sensitive environments requiring air-gapped operation +## License Compliance -#### Privacy Mode Behavior -| Feature | Privacy Mode ON | Privacy Mode OFF | -|---------|----------------|------------------| -| External API calls | ❌ Disabled | ✅ Configurable | -| OpenFlights lookup | ✅ Enabled | ✅ Enabled | -| Callsign caching | ✅ Local only | ✅ Full caching | -| Data transmission | ❌ None | ⚠️ API calls only | - -#### Use Cases for Privacy Mode -- **Military installations**: No external data transmission allowed -- **Air-gapped networks**: No internet connectivity available -- **Corporate policies**: External API usage prohibited -- **Personal privacy**: User preference for local-only operation - -### Security Considerations - -#### File Permissions -- Database files readable only by skyview user/group -- Configuration files protected from unauthorized access -- Backup files inherit secure permissions - -#### Data Protection -- Local SQLite database with file-system level security -- No cloud storage or external database dependencies -- All aviation data processed and stored locally - -#### Network Security -- External API calls (when enabled) use HTTPS only -- No persistent connections to external services -- Optional certificate validation for API endpoints - -### Data Integrity -- Foreign key constraints where applicable -- Transaction isolation for concurrent operations -- Checksums for migration verification +SkyView maintains strict license separation: +- **SkyView binary**: Contains no external data (stays MIT licensed) +- **Runtime import**: Users choose which sources to import +- **Safe defaults**: Only public domain sources updated automatically +- **User choice**: Each person decides their own license compatibility ## Troubleshooting -### Common Issues - -#### Database Locked -``` -Error: database is locked -``` -**Solution**: Stop SkyView service, check for stale lock files, restart - -#### Migration Failures -``` -Error: migration 3 failed: table already exists -``` -**Solution**: Check schema version, restore from backup, retry migration - -#### Permission Denied -``` -Error: unable to open database file -``` -**Solution**: Verify file permissions, check directory ownership, ensure disk space - -### Diagnostic Commands +### Check Service Status ```bash -# Check database integrity -sqlite3 /var/lib/skyview/skyview.db "PRAGMA integrity_check;" - -# View schema version -sqlite3 /var/lib/skyview/skyview.db "SELECT * FROM schema_info;" - -# Database statistics -sqlite3 /var/lib/skyview/skyview.db ".dbinfo" +systemctl status skyview-database-update.timer +journalctl -u skyview-database-update.service -f ``` -## Testing and Quality Assurance - -SkyView includes comprehensive test coverage for all database functionality to ensure reliability and data integrity. - -### Test Coverage Areas - -#### Core Database Functionality -- **Database Creation and Initialization**: Connection management, configuration handling -- **Migration System**: Schema versioning, upgrade/downgrade operations -- **Connection Pooling**: Concurrent access, connection lifecycle management -- **SQLite Pragma Settings**: WAL mode, foreign keys, performance optimizations - -#### Data Loading and Management -- **Multi-Source Loading**: OpenFlights, OurAirports data integration -- **Conflict Resolution**: Upsert operations, duplicate handling -- **Error Handling**: Network failures, malformed data recovery -- **Performance Validation**: Loading speed, memory usage optimization - -#### Callsign Enhancement System -- **Parsing Logic**: Callsign validation, airline code extraction -- **Database Integration**: Local lookups, caching operations -- **Search Functionality**: Airline filtering, country-based queries -- **Cache Management**: TTL handling, cleanup operations - -#### Optimization System -- **VACUUM Operations**: Space reclamation, performance monitoring -- **Page Size Optimization**: Configuration validation, storage efficiency -- **Statistics Generation**: Metrics accuracy, reporting consistency -- **Maintenance Scheduling**: Automated optimization, interval management - -### Test Infrastructure - -#### Automated Test Setup -```go -// setupTestDatabase creates isolated test environment -func setupTestDatabase(t *testing.T) (*Database, func()) { - tempFile, _ := os.CreateTemp("", "test_skyview_*.db") - config := &Config{Path: tempFile.Name()} - db, _ := NewDatabase(config) - db.Initialize() // Run all migrations - - cleanup := func() { - db.Close() - os.Remove(tempFile.Name()) - } - return db, cleanup -} -``` - -#### Network-Safe Testing -Tests gracefully handle network connectivity issues: -- Skip tests requiring external data sources when offline -- Provide meaningful error messages for connectivity failures -- Use local test data when external sources are unavailable - -### Running Tests - +### Manual Database Reset ```bash -# Run all database tests -go test -v ./internal/database/... - -# Run tests in short mode (skip long-running network tests) -go test -v -short ./internal/database/... - -# Run specific test categories -go test -v -run="TestDatabase" ./internal/database/... -go test -v -run="TestOptimization" ./internal/database/... -go test -v -run="TestCallsign" ./internal/database/... +systemctl stop skyview-database-update.timer +skyview-data reset --force +skyview-data update +systemctl start skyview-database-update.timer ``` -## Future Enhancements +### Permissions Issues +```bash +sudo chown skyview:skyview /var/lib/skyview/ +sudo chmod 755 /var/lib/skyview/ +``` -### Planned Features -- **Compression**: Time-series compression for long-term storage -- **Partitioning**: Date-based partitioning for large datasets -- **Replication**: Read replica support for high-availability setups -- **Analytics**: Built-in reporting and statistics tables -- **Enhanced Route Data**: Integration with additional flight tracking APIs -- **Geographic Indexing**: Spatial queries for airport proximity searches +## Files and Directories -### Migration Path -- All enhancements will use versioned migrations -- Backward compatibility maintained for existing installations -- Data preservation prioritized over schema optimization -- Comprehensive testing required for all schema changes \ No newline at end of file +- `/usr/bin/skyview-data` - Database management command +- `/var/lib/skyview/skyview.db` - Database file +- `/usr/share/skyview/scripts/update-database.sh` - Cron helper script +- `/lib/systemd/system/skyview-database-update.*` - Systemd timer files + +For detailed information, see `man skyview-data`. \ No newline at end of file diff --git a/docs/DATABASE.md b/docs/DATABASE.md index 326f1cf..280d603 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -49,7 +49,7 @@ Stores time-series aircraft position and message data: ```sql CREATE TABLE aircraft_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, - icao TEXT NOT NULL, + icao_hex TEXT NOT NULL, timestamp TIMESTAMP NOT NULL, latitude REAL, longitude REAL, @@ -59,8 +59,9 @@ CREATE TABLE aircraft_history ( vertical_rate INTEGER, squawk TEXT, callsign TEXT, - source_id TEXT NOT NULL, - signal_strength REAL + source_id TEXT, + signal_strength REAL, + message_count INTEGER DEFAULT 1 ); ``` @@ -70,123 +71,66 @@ CREATE TABLE aircraft_history ( - `idx_aircraft_history_callsign`: Callsign-based searches #### `airlines` -Multi-source airline database with unified schema: +OpenFlights embedded airline database: ```sql CREATE TABLE airlines ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, alias TEXT, - iata_code TEXT, - icao_code TEXT, + iata TEXT, + icao TEXT, callsign TEXT, country TEXT, - country_code TEXT, - active BOOLEAN DEFAULT 1, - data_source TEXT NOT NULL DEFAULT 'unknown', - source_id TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + active BOOLEAN DEFAULT 1 ); ``` **Indexes:** -- `idx_airlines_icao_code`: ICAO code lookup (primary for callsign enhancement) -- `idx_airlines_iata_code`: IATA code lookup -- `idx_airlines_callsign`: Radio callsign lookup -- `idx_airlines_country_code`: Country-based filtering -- `idx_airlines_active`: Active airlines filtering -- `idx_airlines_source`: Data source tracking +- `idx_airlines_icao`: ICAO code lookup (primary for callsign enhancement) +- `idx_airlines_iata`: IATA code lookup #### `airports` -Multi-source airport database with comprehensive metadata: +OpenFlights embedded airport database: ```sql CREATE TABLE airports ( id INTEGER PRIMARY KEY, name TEXT NOT NULL, - ident TEXT, - type TEXT, city TEXT, - municipality TEXT, - region TEXT, country TEXT, - country_code TEXT, - continent TEXT, - iata_code TEXT, - icao_code TEXT, - local_code TEXT, - gps_code TEXT, + iata TEXT, + icao TEXT, latitude REAL, longitude REAL, - elevation_ft INTEGER, - scheduled_service BOOLEAN DEFAULT 0, - home_link TEXT, - wikipedia_link TEXT, - keywords TEXT, + altitude INTEGER, timezone_offset REAL, - timezone TEXT, dst_type TEXT, - data_source TEXT NOT NULL DEFAULT 'unknown', - source_id TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + timezone TEXT ); ``` **Indexes:** -- `idx_airports_icao_code`: ICAO code lookup -- `idx_airports_iata_code`: IATA code lookup -- `idx_airports_ident`: Airport identifier lookup -- `idx_airports_country_code`: Country-based filtering -- `idx_airports_type`: Airport type filtering -- `idx_airports_coords`: Geographic coordinate queries -- `idx_airports_source`: Data source tracking +- `idx_airports_icao`: ICAO code lookup +- `idx_airports_iata`: IATA code lookup #### `callsign_cache` -Caches external API lookups and local enrichment for callsign enhancement: +Caches external API lookups for callsign enhancement: ```sql CREATE TABLE callsign_cache ( callsign TEXT PRIMARY KEY, airline_icao TEXT, - airline_iata TEXT, airline_name TEXT, - airline_country TEXT, flight_number TEXT, - origin_iata TEXT, -- Departure airport IATA code - destination_iata TEXT, -- Arrival airport IATA code + origin_iata TEXT, + destination_iata TEXT, aircraft_type TEXT, - route TEXT, -- Full route description - status TEXT, -- Flight status (scheduled, delayed, etc.) - source TEXT NOT NULL DEFAULT 'local', cached_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - expires_at TIMESTAMP NOT NULL + expires_at TIMESTAMP, + source TEXT DEFAULT 'local' ); ``` -**Route Information Fields:** -- **`origin_iata`**: IATA code of departure airport (e.g., "JFK" for New York JFK) -- **`destination_iata`**: IATA code of arrival airport (e.g., "LAX" for Los Angeles) -- **`route`**: Human-readable route description (e.g., "JFK-LAX" or "New York to Los Angeles") -- **`status`**: Current flight status when available from external APIs - -These fields enable enhanced flight tracking with origin-destination pairs and route visualization. - **Indexes:** - `idx_callsign_cache_expires`: Efficient cache cleanup -- `idx_callsign_cache_airline`: Airline-based queries - -#### `data_sources` -Tracks loaded external data sources and their metadata: -```sql -CREATE TABLE data_sources ( - name TEXT PRIMARY KEY, - license TEXT NOT NULL, - url TEXT, - version TEXT, - imported_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - record_count INTEGER DEFAULT 0, - user_accepted_license BOOLEAN DEFAULT 0 -); -``` ## Database Location Strategy @@ -251,72 +195,15 @@ var migrations = []Migration{ } ``` -## Data Sources and Loading - -SkyView supports multiple aviation data sources with automatic conflict resolution and license compliance. - -### Supported Data Sources - -#### OpenFlights Airlines Database -- **Source**: https://openflights.org/data.html -- **License**: Open Database License (ODbL) 1.0 -- **Content**: Global airline data with ICAO/IATA codes, callsigns, and country information -- **Records**: ~6,162 airlines -- **Update Method**: Runtime download (no license confirmation required) - -#### OpenFlights Airports Database -- **Source**: https://openflights.org/data.html -- **License**: Open Database License (ODbL) 1.0 -- **Content**: Global airport data with coordinates, codes, and metadata -- **Records**: ~7,698 airports -- **Update Method**: Runtime download - -#### OurAirports Database -- **Source**: https://ourairports.com/data/ -- **License**: Creative Commons Zero (CC0) 1.0 -- **Content**: Comprehensive airport database with detailed metadata -- **Records**: ~83,557 airports -- **Update Method**: Runtime download - -### Data Loading System - -#### Intelligent Conflict Resolution -The data loading system uses **INSERT OR REPLACE** upserts to handle overlapping data: - -```sql -INSERT OR REPLACE INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) -``` - -This ensures that: -- Duplicate records are automatically updated rather than causing errors -- Later data sources can override earlier ones -- Database integrity is maintained during bulk loads - -#### Loading Process -1. **Source Validation**: Verify data source accessibility and format -2. **Incremental Processing**: Process data in chunks to manage memory -3. **Error Handling**: Log and continue on individual record errors -4. **Statistics Reporting**: Track records processed, added, and errors -5. **Source Tracking**: Record metadata about each loaded source - -#### Performance Characteristics -- **OpenFlights Airlines**: ~6,162 records in ~363ms -- **OpenFlights Airports**: ~7,698 records in ~200ms -- **OurAirports**: ~83,557 records in ~980ms -- **Error Rate**: <0.1% under normal conditions - ## Configuration Integration ### Database Configuration ```json { "database": { - "path": "/var/lib/skyview-adsb/skyview.db", + "path": "/var/lib/skyview/skyview.db", "max_history_days": 7, - "backup_on_upgrade": true, - "vacuum_interval": "24h", - "page_size": 4096 + "backup_on_upgrade": true }, "callsign": { "enabled": true, @@ -407,26 +294,10 @@ This ensures that: 4. **Service**: Systemd integration with proper database access ### Service User -- User: `skyview-adsb` -- Home: `/var/lib/skyview-adsb` +- User: `skyview` +- Home: `/var/lib/skyview` - Shell: `/bin/false` (service account) -- Database: Read/write access to `/var/lib/skyview-adsb/` - -### Automatic Database Updates -The systemd service configuration includes automatic database updates on startup: - -```ini -[Service] -Type=simple -User=skyview-adsb -Group=skyview-adsb -# Update database before starting main service -ExecStartPre=/usr/bin/skyview-data -config /etc/skyview-adsb/config.json update -TimeoutStartSec=300 -ExecStart=/usr/bin/skyview -config /etc/skyview-adsb/config.json -``` - -This ensures aviation data sources are refreshed before each service start, complementing the weekly timer-based updates. +- Database: Read/write access to `/var/lib/skyview/` ## Data Retention and Cleanup @@ -449,89 +320,6 @@ WHERE expires_at < datetime('now'); VACUUM; ``` -## Database Optimization - -SkyView includes a comprehensive database optimization system that automatically manages storage efficiency and performance. - -### Optimization Features - -#### Automatic VACUUM Operations -- **Full VACUUM**: Rebuilds database to reclaim deleted space -- **Incremental VACUUM**: Gradual space reclamation with minimal performance impact -- **Scheduled Maintenance**: Configurable intervals for automatic optimization -- **Size Reporting**: Before/after statistics with space savings metrics - -#### Storage Optimization -- **Page Size Optimization**: Configurable SQLite page size for optimal performance -- **Auto-Vacuum Configuration**: Enables incremental space reclamation -- **Statistics Updates**: ANALYZE operations for query plan optimization -- **Efficiency Monitoring**: Real-time storage efficiency reporting - -### Using the Optimization System - -#### Command Line Interface -```bash -# Run comprehensive database optimization -skyview-data optimize - -# Run with force flag to skip confirmation prompts -skyview-data optimize --force - -# Check current optimization statistics -skyview-data optimize --stats-only -``` - -#### Optimization Output Example -``` -Optimizing database for storage efficiency... -✓ Auto VACUUM: Enable incremental auto-vacuum -✓ Incremental VACUUM: Reclaim free pages incrementally -✓ Optimize: Update SQLite query planner statistics -✓ Analyze: Update table statistics for better query plans - -VACUUM completed in 1.2s: 275.3 MB → 263.1 MB (saved 12.2 MB, 4.4%) - -Database optimization completed successfully. -Storage efficiency: 96.8% (263.1 MB used of 272.4 MB allocated) -``` - -#### Configuration Options -```json -{ - "database": { - "vacuum_interval": "24h", - "page_size": 4096, - "enable_compression": true, - "compression_level": 6 - } -} -``` - -### Optimization Statistics - -The optimization system provides detailed metrics about database performance: - -#### Available Statistics -- **Database Size**: Total file size in bytes -- **Page Statistics**: Page size, count, and utilization -- **Storage Efficiency**: Percentage of allocated space actually used -- **Free Space**: Amount of reclaimable space available -- **Auto-Vacuum Status**: Current auto-vacuum configuration -- **Last Optimization**: Timestamp of most recent optimization - -#### Programmatic Access -```go -// Get current optimization statistics -optimizer := NewOptimizationManager(db, config) -stats, err := optimizer.GetOptimizationStats() -if err != nil { - log.Fatal("Failed to get stats:", err) -} - -fmt.Printf("Database efficiency: %.1f%%\n", stats.Efficiency) -fmt.Printf("Storage used: %.1f MB\n", float64(stats.DatabaseSize)/(1024*1024)) -``` - ## Performance Considerations ### Query Optimization @@ -541,10 +329,8 @@ fmt.Printf("Storage used: %.1f MB\n", float64(stats.DatabaseSize)/(1024*1024)) ### Storage Efficiency - Configurable history limits prevent unbounded growth -- Automatic VACUUM operations with optimization reporting +- Periodic VACUUM operations reclaim deleted space - Compressed timestamps and efficient data types -- Page size optimization for storage efficiency -- Auto-vacuum configuration for incremental space reclamation ### Memory Usage - WAL mode for concurrent read/write access @@ -642,76 +428,6 @@ sqlite3 /var/lib/skyview/skyview.db "SELECT * FROM schema_info;" sqlite3 /var/lib/skyview/skyview.db ".dbinfo" ``` -## Testing and Quality Assurance - -SkyView includes comprehensive test coverage for all database functionality to ensure reliability and data integrity. - -### Test Coverage Areas - -#### Core Database Functionality -- **Database Creation and Initialization**: Connection management, configuration handling -- **Migration System**: Schema versioning, upgrade/downgrade operations -- **Connection Pooling**: Concurrent access, connection lifecycle management -- **SQLite Pragma Settings**: WAL mode, foreign keys, performance optimizations - -#### Data Loading and Management -- **Multi-Source Loading**: OpenFlights, OurAirports data integration -- **Conflict Resolution**: Upsert operations, duplicate handling -- **Error Handling**: Network failures, malformed data recovery -- **Performance Validation**: Loading speed, memory usage optimization - -#### Callsign Enhancement System -- **Parsing Logic**: Callsign validation, airline code extraction -- **Database Integration**: Local lookups, caching operations -- **Search Functionality**: Airline filtering, country-based queries -- **Cache Management**: TTL handling, cleanup operations - -#### Optimization System -- **VACUUM Operations**: Space reclamation, performance monitoring -- **Page Size Optimization**: Configuration validation, storage efficiency -- **Statistics Generation**: Metrics accuracy, reporting consistency -- **Maintenance Scheduling**: Automated optimization, interval management - -### Test Infrastructure - -#### Automated Test Setup -```go -// setupTestDatabase creates isolated test environment -func setupTestDatabase(t *testing.T) (*Database, func()) { - tempFile, _ := os.CreateTemp("", "test_skyview_*.db") - config := &Config{Path: tempFile.Name()} - db, _ := NewDatabase(config) - db.Initialize() // Run all migrations - - cleanup := func() { - db.Close() - os.Remove(tempFile.Name()) - } - return db, cleanup -} -``` - -#### Network-Safe Testing -Tests gracefully handle network connectivity issues: -- Skip tests requiring external data sources when offline -- Provide meaningful error messages for connectivity failures -- Use local test data when external sources are unavailable - -### Running Tests - -```bash -# Run all database tests -go test -v ./internal/database/... - -# Run tests in short mode (skip long-running network tests) -go test -v -short ./internal/database/... - -# Run specific test categories -go test -v -run="TestDatabase" ./internal/database/... -go test -v -run="TestOptimization" ./internal/database/... -go test -v -run="TestCallsign" ./internal/database/... -``` - ## Future Enhancements ### Planned Features @@ -719,11 +435,8 @@ go test -v -run="TestCallsign" ./internal/database/... - **Partitioning**: Date-based partitioning for large datasets - **Replication**: Read replica support for high-availability setups - **Analytics**: Built-in reporting and statistics tables -- **Enhanced Route Data**: Integration with additional flight tracking APIs -- **Geographic Indexing**: Spatial queries for airport proximity searches ### Migration Path - All enhancements will use versioned migrations - Backward compatibility maintained for existing installations -- Data preservation prioritized over schema optimization -- Comprehensive testing required for all schema changes \ No newline at end of file +- Data preservation prioritized over schema optimization \ No newline at end of file diff --git a/go.mod b/go.mod index 50409e4..24b62db 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,4 @@ require ( github.com/gorilla/websocket v1.5.3 ) -require github.com/mattn/go-sqlite3 v1.14.32 +require github.com/mattn/go-sqlite3 v1.14.32 // indirect diff --git a/internal/database/database.go b/internal/database/database.go index 108a0d9..8a44418 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -43,11 +43,6 @@ type Config struct { // Maintenance settings VacuumInterval time.Duration `json:"vacuum_interval"` // Default: 24 hours CleanupInterval time.Duration `json:"cleanup_interval"` // Default: 1 hour - - // Compression settings - EnableCompression bool `json:"enable_compression"` // Enable automatic compression - CompressionLevel int `json:"compression_level"` // Compression level (1-9, default: 6) - PageSize int `json:"page_size"` // SQLite page size (default: 4096) } // AircraftHistoryRecord represents a stored aircraft position update diff --git a/internal/database/database_test.go b/internal/database/database_test.go deleted file mode 100644 index 4e6d21b..0000000 --- a/internal/database/database_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package database - -import ( - "os" - "testing" - "time" -) - -func TestNewDatabase(t *testing.T) { - // Create temporary database file - tempFile, err := os.CreateTemp("", "test_skyview_*.db") - if err != nil { - t.Fatal("Failed to create temp database file:", err) - } - defer os.Remove(tempFile.Name()) - tempFile.Close() - - config := &Config{ - Path: tempFile.Name(), - VacuumInterval: time.Hour, - } - - db, err := NewDatabase(config) - if err != nil { - t.Fatal("Failed to create database:", err) - } - defer db.Close() - - if db == nil { - t.Fatal("NewDatabase() returned nil") - } - - // Test connection - conn := db.GetConnection() - if conn == nil { - t.Fatal("GetConnection() returned nil") - } - - // Test basic query - var result int - err = conn.QueryRow("SELECT 1").Scan(&result) - if err != nil { - t.Error("Basic query failed:", err) - } - if result != 1 { - t.Error("Basic query returned wrong result:", result) - } -} - -func TestDatabaseClose(t *testing.T) { - tempFile, err := os.CreateTemp("", "test_skyview_*.db") - if err != nil { - t.Fatal("Failed to create temp database file:", err) - } - defer os.Remove(tempFile.Name()) - tempFile.Close() - - config := &Config{Path: tempFile.Name()} - db, err := NewDatabase(config) - if err != nil { - t.Fatal("Failed to create database:", err) - } - - // Close should not error - if err := db.Close(); err != nil { - t.Error("Database Close() returned error:", err) - } - - // Second close should be safe - if err := db.Close(); err != nil { - t.Error("Second Close() returned error:", err) - } - - // Connection should be nil after close - conn := db.GetConnection() - if conn != nil { - t.Error("GetConnection() should return nil after Close()") - } -} - -func TestDatabaseConfig(t *testing.T) { - tempFile, err := os.CreateTemp("", "test_skyview_*.db") - if err != nil { - t.Fatal("Failed to create temp database file:", err) - } - defer os.Remove(tempFile.Name()) - tempFile.Close() - - config := &Config{ - Path: tempFile.Name(), - VacuumInterval: 2 * time.Hour, - } - - db, err := NewDatabase(config) - if err != nil { - t.Fatal("Failed to create database:", err) - } - defer db.Close() - - // Test that config is stored correctly - if db.config != config { - t.Error("Database config not stored correctly") - } - if db.config.VacuumInterval != 2*time.Hour { - t.Error("VacuumInterval not preserved:", db.config.VacuumInterval) - } -} - -func TestDatabaseMigrations(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - conn := db.GetConnection() - - // Check that essential tables exist after migrations - tables := []string{"airlines", "airports", "callsign_cache", "data_sources", "aircraft_history"} - for _, table := range tables { - var count int - query := "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?" - err := conn.QueryRow(query, table).Scan(&count) - if err != nil { - t.Errorf("Failed to check for table %s: %v", table, err) - } - if count != 1 { - t.Errorf("Table %s does not exist", table) - } - } -} - -func TestDatabasePragmas(t *testing.T) { - tempFile, err := os.CreateTemp("", "test_skyview_*.db") - if err != nil { - t.Fatal("Failed to create temp database file:", err) - } - defer os.Remove(tempFile.Name()) - tempFile.Close() - - config := &Config{Path: tempFile.Name()} - db, err := NewDatabase(config) - if err != nil { - t.Fatal("Failed to create database:", err) - } - defer db.Close() - - conn := db.GetConnection() - - // Check that foreign keys are enabled - var foreignKeys int - err = conn.QueryRow("PRAGMA foreign_keys").Scan(&foreignKeys) - if err != nil { - t.Error("Failed to check foreign_keys pragma:", err) - } - if foreignKeys != 1 { - t.Error("Foreign keys should be enabled") - } - - // Check journal mode - var journalMode string - err = conn.QueryRow("PRAGMA journal_mode").Scan(&journalMode) - if err != nil { - t.Error("Failed to check journal_mode:", err) - } - // Should be WAL mode for better concurrency - if journalMode != "wal" { - t.Errorf("Expected WAL journal mode, got: %s", journalMode) - } -} \ No newline at end of file diff --git a/internal/database/loader.go b/internal/database/loader.go index 6d31dfd..04b4653 100644 --- a/internal/database/loader.go +++ b/internal/database/loader.go @@ -79,7 +79,7 @@ func GetAvailableDataSources() []DataSource { Name: "OpenFlights Airlines", License: "AGPL-3.0", URL: "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat", - RequiresConsent: false, // Runtime data consumption doesn't require explicit consent + RequiresConsent: true, Format: "openflights", Version: "latest", }, @@ -87,7 +87,7 @@ func GetAvailableDataSources() []DataSource { Name: "OpenFlights Airports", License: "AGPL-3.0", URL: "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat", - RequiresConsent: false, // Runtime data consumption doesn't require explicit consent + RequiresConsent: true, Format: "openflights", Version: "latest", }, @@ -169,7 +169,7 @@ func (dl *DataLoader) loadOpenFlightsAirlines(reader io.Reader, source DataSourc csvReader.FieldsPerRecord = -1 // Variable number of fields insertStmt, err := tx.Prepare(` - INSERT OR REPLACE INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) + INSERT INTO airlines (id, name, alias, iata, icao, callsign, country, active, data_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `) if err != nil { @@ -255,8 +255,8 @@ func (dl *DataLoader) loadOpenFlightsAirports(reader io.Reader, source DataSourc csvReader.FieldsPerRecord = -1 insertStmt, err := tx.Prepare(` - INSERT OR REPLACE INTO airports (id, name, city, country, iata_code, icao_code, latitude, longitude, - elevation_ft, timezone_offset, dst_type, timezone, data_source) + INSERT INTO airports (id, name, city, country, iata, icao, latitude, longitude, + altitude, timezone_offset, dst_type, timezone, data_source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) if err != nil { diff --git a/internal/database/loader_test.go b/internal/database/loader_test.go deleted file mode 100644 index 9bfebe8..0000000 --- a/internal/database/loader_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package database - -import ( - "strings" - "testing" -) - -func TestDataLoader_Creation(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - loader := NewDataLoader(db.GetConnection()) - if loader == nil { - t.Fatal("NewDataLoader returned nil") - } -} - -func TestDataLoader_LoadOpenFlightsAirlines(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - loader := NewDataLoader(db.GetConnection()) - - // Create a test data source for OpenFlights Airlines - source := DataSource{ - Name: "OpenFlights Airlines Test", - License: "ODbL 1.0", - URL: "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat", - Format: "openflights", - Version: "2024-test", - } - - result, err := loader.LoadDataSource(source) - if err != nil { - // Network issues in tests are acceptable - if strings.Contains(err.Error(), "connection") || - strings.Contains(err.Error(), "timeout") || - strings.Contains(err.Error(), "no such host") { - t.Skipf("Skipping network test due to connectivity issue: %v", err) - } - t.Fatal("LoadDataSource failed:", err) - } - - if result == nil { - t.Fatal("Expected load result, got nil") - } - - t.Logf("Loaded airlines: Total=%d, New=%d, Errors=%d, Duration=%v", - result.RecordsTotal, result.RecordsNew, result.RecordsError, result.Duration) - - // Verify some data was processed - if result.RecordsTotal == 0 { - t.Error("No records were processed") - } -} - -func TestDataLoader_LoadOurAirports(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - loader := NewDataLoader(db.GetConnection()) - - // Create a test data source for OurAirports - source := DataSource{ - Name: "OurAirports Test", - License: "CC0 1.0", - URL: "https://davidmegginson.github.io/ourairports-data/airports.csv", - Format: "ourairports", - Version: "2024-test", - } - - result, err := loader.LoadDataSource(source) - if err != nil { - // Network issues in tests are acceptable - if strings.Contains(err.Error(), "connection") || - strings.Contains(err.Error(), "timeout") || - strings.Contains(err.Error(), "no such host") { - t.Skipf("Skipping network test due to connectivity issue: %v", err) - } - t.Fatal("LoadDataSource failed:", err) - } - - if result != nil { - t.Logf("Loaded airports: Total=%d, New=%d, Errors=%d, Duration=%v", - result.RecordsTotal, result.RecordsNew, result.RecordsError, result.Duration) - } -} - -func TestDataLoader_GetLoadedDataSources(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - loader := NewDataLoader(db.GetConnection()) - - sources, err := loader.GetLoadedDataSources() - if err != nil { - t.Fatal("GetLoadedDataSources failed:", err) - } - - // Initially should be empty or minimal - t.Logf("Found %d loaded data sources", len(sources)) -} - -func TestDataLoader_ClearDataSource(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - loader := NewDataLoader(db.GetConnection()) - - // Test clearing a non-existent source (should not error) - err := loader.ClearDataSource("nonexistent") - if err != nil { - t.Error("ClearDataSource should not error on nonexistent source:", err) - } -} - -func TestDataSource_Struct(t *testing.T) { - source := DataSource{ - Name: "Test Source", - License: "Test License", - URL: "https://example.com/data.csv", - RequiresConsent: false, - UserAcceptedLicense: true, - Format: "csv", - Version: "1.0", - } - - // Test that all fields are accessible - if source.Name != "Test Source" { - t.Error("Name field not preserved") - } - if source.License != "Test License" { - t.Error("License field not preserved") - } - if source.URL != "https://example.com/data.csv" { - t.Error("URL field not preserved") - } - if source.RequiresConsent != false { - t.Error("RequiresConsent field not preserved") - } - if source.UserAcceptedLicense != true { - t.Error("UserAcceptedLicense field not preserved") - } - if source.Format != "csv" { - t.Error("Format field not preserved") - } - if source.Version != "1.0" { - t.Error("Version field not preserved") - } -} - -func TestLoadResult_Struct(t *testing.T) { - result := LoadResult{ - Source: "Test Source", - RecordsTotal: 100, - RecordsNew: 80, - RecordsError: 5, - Errors: []string{"error1", "error2"}, - } - - // Test that all fields are accessible - if result.Source != "Test Source" { - t.Error("Source field not preserved") - } - if result.RecordsTotal != 100 { - t.Error("RecordsTotal field not preserved") - } - if result.RecordsNew != 80 { - t.Error("RecordsNew field not preserved") - } - if result.RecordsError != 5 { - t.Error("RecordsError field not preserved") - } - if len(result.Errors) != 2 { - t.Error("Errors field not preserved") - } -} \ No newline at end of file diff --git a/internal/database/manager_callsign_test.go b/internal/database/manager_callsign_test.go deleted file mode 100644 index 731f61a..0000000 --- a/internal/database/manager_callsign_test.go +++ /dev/null @@ -1,268 +0,0 @@ -package database - -import ( - "testing" -) - -func TestCallsignManager_Creation(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - if manager == nil { - t.Fatal("NewCallsignManager returned nil") - } -} - -func TestCallsignManager_ParseCallsign(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - testCases := []struct { - callsign string - expectedValid bool - expectedAirline string - expectedFlight string - }{ - {"UAL123", true, "UAL", "123"}, - {"BA4567", true, "BA", "4567"}, - {"AFR89", true, "AFR", "89"}, - {"N123AB", false, "", ""}, // Aircraft registration, not callsign - {"INVALID", false, "", ""}, // No numbers - {"123", false, "", ""}, // Only numbers - {"A", false, "", ""}, // Too short - {"", false, "", ""}, // Empty - } - - for _, tc := range testCases { - result := manager.ParseCallsign(tc.callsign) - if result.IsValid != tc.expectedValid { - t.Errorf("ParseCallsign(%s): expected valid=%v, got %v", - tc.callsign, tc.expectedValid, result.IsValid) - } - if result.IsValid && result.AirlineCode != tc.expectedAirline { - t.Errorf("ParseCallsign(%s): expected airline=%s, got %s", - tc.callsign, tc.expectedAirline, result.AirlineCode) - } - if result.IsValid && result.FlightNumber != tc.expectedFlight { - t.Errorf("ParseCallsign(%s): expected flight=%s, got %s", - tc.callsign, tc.expectedFlight, result.FlightNumber) - } - } -} - -func TestCallsignManager_GetCallsignInfo(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - // Insert test airline data - conn := db.GetConnection() - _, err := conn.Exec(` - INSERT INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) - VALUES (1, 'Test Airways', 'Test', 'TA', 'TST', 'TESTAIR', 'United States', 1, 'test') - `) - if err != nil { - t.Fatal("Failed to insert test data:", err) - } - - // Test valid callsign - info, err := manager.GetCallsignInfo("TST123") - if err != nil { - t.Fatal("GetCallsignInfo failed:", err) - } - - if info == nil { - t.Fatal("Expected callsign info, got nil") - } - - if info.OriginalCallsign != "TST123" { - t.Errorf("Expected callsign TST123, got %s", info.OriginalCallsign) - } - if info.AirlineCode != "TST" { - t.Errorf("Expected airline code TST, got %s", info.AirlineCode) - } - if info.FlightNumber != "123" { - t.Errorf("Expected flight number 123, got %s", info.FlightNumber) - } - if info.AirlineName != "Test Airways" { - t.Errorf("Expected airline name 'Test Airways', got %s", info.AirlineName) - } - if info.AirlineCountry != "United States" { - t.Errorf("Expected airline country 'United States', got %s", info.AirlineCountry) - } -} - -func TestCallsignManager_GetCallsignInfo_InvalidCallsign(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - // Test with invalid callsign format - info, err := manager.GetCallsignInfo("INVALID") - if err != nil { - t.Error("GetCallsignInfo should not error on invalid format:", err) - } - if info == nil { - t.Fatal("Expected info structure even for invalid callsign") - } - if info.IsValid { - t.Error("Invalid callsign should not be marked as valid") - } - - // Test with unknown airline - info, err = manager.GetCallsignInfo("UNK123") - if err != nil { - t.Error("GetCallsignInfo should not error on unknown airline:", err) - } - if info == nil { - t.Fatal("Expected info structure for unknown airline") - } -} - -func TestCallsignManager_GetCallsignInfo_EmptyCallsign(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - // Test with empty callsign - info, err := manager.GetCallsignInfo("") - if err == nil { - t.Error("GetCallsignInfo should error on empty callsign") - } - if info != nil { - t.Error("Expected nil info for empty callsign") - } -} - -func TestCallsignManager_ClearExpiredCache(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - err := manager.ClearExpiredCache() - if err != nil { - t.Error("ClearExpiredCache should not error:", err) - } -} - -func TestCallsignManager_GetCacheStats(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - stats, err := manager.GetCacheStats() - if err != nil { - t.Error("GetCacheStats should not error:", err) - } - - if stats == nil { - t.Error("Expected cache stats, got nil") - } - - t.Logf("Cache stats: %+v", stats) -} - -func TestCallsignManager_SearchAirlines(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - // Insert test airline data - conn := db.GetConnection() - _, err := conn.Exec(` - INSERT INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) - VALUES (1, 'Test Airways', 'Test', 'TA', 'TST', 'TESTAIR', 'United States', 1, 'test'), - (2, 'Another Airline', 'Another', 'AA', 'ANO', 'ANOTHER', 'Canada', 1, 'test') - `) - if err != nil { - t.Fatal("Failed to insert test data:", err) - } - - // Search for airlines - airlines, err := manager.SearchAirlines("Test") - if err != nil { - t.Fatal("SearchAirlines failed:", err) - } - - found := false - for _, airline := range airlines { - if airline.Name == "Test Airways" { - found = true - break - } - } - if !found { - t.Error("Expected to find Test Airways in search results") - } - - t.Logf("Found %d airlines matching 'Test'", len(airlines)) -} - -func TestCallsignManager_GetAirlinesByCountry(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - manager := NewCallsignManager(db.GetConnection()) - - // Insert test airline data - conn := db.GetConnection() - _, err := conn.Exec(` - INSERT INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) - VALUES (1, 'US Airways', 'US', 'US', 'USA', 'USAIR', 'United States', 1, 'test'), - (2, 'Canada Air', 'CA', 'CA', 'CAN', 'CANAIR', 'Canada', 1, 'test') - `) - if err != nil { - t.Fatal("Failed to insert test data:", err) - } - - // Get airlines by country - airlines, err := manager.GetAirlinesByCountry("United States") - if err != nil { - t.Fatal("GetAirlinesByCountry failed:", err) - } - - found := false - for _, airline := range airlines { - if airline.Name == "US Airways" { - found = true - break - } - } - if !found { - t.Error("Expected to find US Airways for United States") - } - - t.Logf("Found %d airlines in United States", len(airlines)) -} - -func TestCallsignParseResult_Struct(t *testing.T) { - result := &CallsignParseResult{ - OriginalCallsign: "UAL123", - AirlineCode: "UAL", - FlightNumber: "123", - IsValid: true, - } - - // Test that all fields are accessible - if result.OriginalCallsign != "UAL123" { - t.Error("OriginalCallsign field not preserved") - } - if result.AirlineCode != "UAL" { - t.Error("AirlineCode field not preserved") - } - if result.FlightNumber != "123" { - t.Error("FlightNumber field not preserved") - } - if !result.IsValid { - t.Error("IsValid field not preserved") - } -} \ No newline at end of file diff --git a/internal/database/optimization.go b/internal/database/optimization.go deleted file mode 100644 index abe322e..0000000 --- a/internal/database/optimization.go +++ /dev/null @@ -1,208 +0,0 @@ -package database - -import ( - "fmt" - "os" - "time" -) - -// OptimizationManager handles database storage optimization using SQLite built-in features -type OptimizationManager struct { - db *Database - config *Config - lastVacuum time.Time -} - -// NewOptimizationManager creates a new optimization manager -func NewOptimizationManager(db *Database, config *Config) *OptimizationManager { - return &OptimizationManager{ - db: db, - config: config, - } -} - -// PerformMaintenance runs database maintenance tasks including VACUUM -func (om *OptimizationManager) PerformMaintenance() error { - now := time.Now() - - // Check if VACUUM is needed - if om.config.VacuumInterval > 0 && now.Sub(om.lastVacuum) >= om.config.VacuumInterval { - if err := om.VacuumDatabase(); err != nil { - return fmt.Errorf("vacuum failed: %w", err) - } - om.lastVacuum = now - } - - return nil -} - -// VacuumDatabase performs VACUUM to reclaim space and optimize database -func (om *OptimizationManager) VacuumDatabase() error { - conn := om.db.GetConnection() - if conn == nil { - return fmt.Errorf("database connection not available") - } - - start := time.Now() - - // Get size before VACUUM - sizeBefore, err := om.getDatabaseSize() - if err != nil { - return fmt.Errorf("failed to get database size: %w", err) - } - - // Perform VACUUM - if _, err := conn.Exec("VACUUM"); err != nil { - return fmt.Errorf("VACUUM operation failed: %w", err) - } - - // Get size after VACUUM - sizeAfter, err := om.getDatabaseSize() - if err != nil { - return fmt.Errorf("failed to get database size after VACUUM: %w", err) - } - - duration := time.Since(start) - savedBytes := sizeBefore - sizeAfter - savedPercent := float64(savedBytes) / float64(sizeBefore) * 100 - - fmt.Printf("VACUUM completed in %v: %.1f MB → %.1f MB (saved %.1f MB, %.1f%%)\n", - duration, - float64(sizeBefore)/(1024*1024), - float64(sizeAfter)/(1024*1024), - float64(savedBytes)/(1024*1024), - savedPercent) - - return nil -} - -// OptimizeDatabase applies various SQLite optimizations for better storage efficiency -func (om *OptimizationManager) OptimizeDatabase() error { - conn := om.db.GetConnection() - if conn == nil { - return fmt.Errorf("database connection not available") - } - - fmt.Println("Optimizing database for storage efficiency...") - - // Apply storage-friendly pragmas - optimizations := []struct{ - name string - query string - description string - }{ - {"Auto VACUUM", "PRAGMA auto_vacuum = INCREMENTAL", "Enable incremental auto-vacuum"}, - {"Incremental VACUUM", "PRAGMA incremental_vacuum", "Reclaim free pages incrementally"}, - {"Optimize", "PRAGMA optimize", "Update SQLite query planner statistics"}, - {"Analyze", "ANALYZE", "Update table statistics for better query plans"}, - } - - for _, opt := range optimizations { - if _, err := conn.Exec(opt.query); err != nil { - fmt.Printf("Warning: %s failed: %v\n", opt.name, err) - } else { - fmt.Printf("✓ %s: %s\n", opt.name, opt.description) - } - } - - return nil -} - -// OptimizePageSize sets an optimal page size for the database (requires rebuild) -func (om *OptimizationManager) OptimizePageSize(pageSize int) error { - conn := om.db.GetConnection() - if conn == nil { - return fmt.Errorf("database connection not available") - } - - // Check current page size - var currentPageSize int - if err := conn.QueryRow("PRAGMA page_size").Scan(¤tPageSize); err != nil { - return fmt.Errorf("failed to get current page size: %w", err) - } - - if currentPageSize == pageSize { - fmt.Printf("Page size already optimal: %d bytes\n", pageSize) - return nil - } - - fmt.Printf("Optimizing page size: %d → %d bytes (requires VACUUM)\n", currentPageSize, pageSize) - - // Set new page size - query := fmt.Sprintf("PRAGMA page_size = %d", pageSize) - if _, err := conn.Exec(query); err != nil { - return fmt.Errorf("failed to set page size: %w", err) - } - - // VACUUM to apply the new page size - if err := om.VacuumDatabase(); err != nil { - return fmt.Errorf("failed to apply page size change: %w", err) - } - - return nil -} - -// GetOptimizationStats returns current database optimization statistics -func (om *OptimizationManager) GetOptimizationStats() (*OptimizationStats, error) { - stats := &OptimizationStats{} - - // Get database size - size, err := om.getDatabaseSize() - if err != nil { - return nil, err - } - stats.DatabaseSize = size - - // Get page statistics - conn := om.db.GetConnection() - if conn != nil { - var pageSize, pageCount, freelistCount int - conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - conn.QueryRow("PRAGMA page_count").Scan(&pageCount) - conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - - stats.PageSize = pageSize - stats.PageCount = pageCount - stats.FreePages = freelistCount - stats.UsedPages = pageCount - freelistCount - - if pageCount > 0 { - stats.Efficiency = float64(stats.UsedPages) / float64(pageCount) * 100 - } - - // Check auto vacuum setting - var autoVacuum int - conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum) - stats.AutoVacuumEnabled = autoVacuum > 0 - } - - stats.LastVacuum = om.lastVacuum - - return stats, nil -} - -// OptimizationStats holds database storage optimization statistics -type OptimizationStats struct { - DatabaseSize int64 `json:"database_size"` - PageSize int `json:"page_size"` - PageCount int `json:"page_count"` - UsedPages int `json:"used_pages"` - FreePages int `json:"free_pages"` - Efficiency float64 `json:"efficiency_percent"` - AutoVacuumEnabled bool `json:"auto_vacuum_enabled"` - LastVacuum time.Time `json:"last_vacuum"` -} - -// getDatabaseSize returns the current database file size in bytes -func (om *OptimizationManager) getDatabaseSize() (int64, error) { - if om.config.Path == "" { - return 0, fmt.Errorf("database path not configured") - } - - stat, err := os.Stat(om.config.Path) - if err != nil { - return 0, fmt.Errorf("failed to stat database file: %w", err) - } - - return stat.Size(), nil -} \ No newline at end of file diff --git a/internal/database/optimization_test.go b/internal/database/optimization_test.go deleted file mode 100644 index c1aeb74..0000000 --- a/internal/database/optimization_test.go +++ /dev/null @@ -1,307 +0,0 @@ -package database - -import ( - "os" - "testing" - "time" -) - -func TestOptimizationManager_VacuumDatabase(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - config := &Config{Path: db.config.Path} - optimizer := NewOptimizationManager(db, config) - - err := optimizer.VacuumDatabase() - if err != nil { - t.Fatal("VacuumDatabase failed:", err) - } - - // Verify vacuum was successful by checking database integrity - conn := db.GetConnection() - var result string - err = conn.QueryRow("PRAGMA integrity_check").Scan(&result) - if err != nil { - t.Error("Failed to run integrity check:", err) - } - if result != "ok" { - t.Errorf("Database integrity check failed: %s", result) - } -} - -func TestOptimizationManager_OptimizeDatabase(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - config := &Config{Path: db.config.Path} - optimizer := NewOptimizationManager(db, config) - - err := optimizer.OptimizeDatabase() - if err != nil { - t.Fatal("OptimizeDatabase failed:", err) - } - - // Check that auto_vacuum was set - conn := db.GetConnection() - var autoVacuum int - err = conn.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum) - if err != nil { - t.Error("Failed to check auto_vacuum setting:", err) - } - // Should be 2 (INCREMENTAL) after optimization - if autoVacuum != 2 { - t.Errorf("Expected auto_vacuum = 2 (INCREMENTAL), got %d", autoVacuum) - } -} - -func TestOptimizationManager_OptimizePageSize(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - config := &Config{Path: db.config.Path} - optimizer := NewOptimizationManager(db, config) - - // Get current page size - conn := db.GetConnection() - var currentPageSize int - err := conn.QueryRow("PRAGMA page_size").Scan(¤tPageSize) - if err != nil { - t.Fatal("Failed to get current page size:", err) - } - - // Set a different page size - targetPageSize := 8192 - if currentPageSize == targetPageSize { - targetPageSize = 4096 // Use different size if already at target - } - - err = optimizer.OptimizePageSize(targetPageSize) - if err != nil { - t.Fatal("OptimizePageSize failed:", err) - } - - // Verify page size was changed - var newPageSize int - err = conn.QueryRow("PRAGMA page_size").Scan(&newPageSize) - if err != nil { - t.Error("Failed to get new page size:", err) - } - if newPageSize != targetPageSize { - t.Errorf("Expected page size %d, got %d", targetPageSize, newPageSize) - } - - // Test setting same page size (should be no-op) - err = optimizer.OptimizePageSize(targetPageSize) - if err != nil { - t.Error("OptimizePageSize failed for same page size:", err) - } -} - -func TestOptimizationManager_GetOptimizationStats(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - config := &Config{Path: db.config.Path} - optimizer := NewOptimizationManager(db, config) - - // Insert some test data to make stats more meaningful - conn := db.GetConnection() - _, err := conn.Exec(` - INSERT INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) - VALUES (1, 'Test Airways', 'Test', 'TA', 'TST', 'TESTAIR', 'United States', 1, 'test') - `) - if err != nil { - t.Error("Failed to insert test data:", err) - } - - stats, err := optimizer.GetOptimizationStats() - if err != nil { - t.Fatal("GetOptimizationStats failed:", err) - } - - if stats == nil { - t.Fatal("Expected stats, got nil") - } - - // Check basic stats - if stats.DatabaseSize <= 0 { - t.Error("Database size should be greater than 0") - } - if stats.PageSize <= 0 { - t.Error("Page size should be greater than 0") - } - if stats.PageCount <= 0 { - t.Error("Page count should be greater than 0") - } - if stats.UsedPages < 0 { - t.Error("Used pages should be non-negative") - } - if stats.FreePages < 0 { - t.Error("Free pages should be non-negative") - } - if stats.Efficiency < 0 || stats.Efficiency > 100 { - t.Errorf("Efficiency should be between 0-100%%, got %.2f%%", stats.Efficiency) - } - - t.Logf("Database stats: Size=%d bytes, Pages=%d (used=%d, free=%d), Efficiency=%.1f%%", - stats.DatabaseSize, stats.PageCount, stats.UsedPages, stats.FreePages, stats.Efficiency) -} - -func TestOptimizationManager_PerformMaintenance(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - config := &Config{ - Path: db.config.Path, - VacuumInterval: time.Millisecond, // Very short interval for testing - } - optimizer := NewOptimizationManager(db, config) - - // Should perform vacuum due to short interval - err := optimizer.PerformMaintenance() - if err != nil { - t.Fatal("PerformMaintenance failed:", err) - } - - // Check that lastVacuum was updated - if optimizer.lastVacuum.IsZero() { - t.Error("lastVacuum should be set after maintenance") - } - - // Wait a bit and run again with longer interval - config.VacuumInterval = time.Hour // Long interval - err = optimizer.PerformMaintenance() - if err != nil { - t.Fatal("Second PerformMaintenance failed:", err) - } -} - -func TestOptimizationManager_getDatabaseSize(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - config := &Config{Path: db.config.Path} - optimizer := NewOptimizationManager(db, config) - - size, err := optimizer.getDatabaseSize() - if err != nil { - t.Fatal("getDatabaseSize failed:", err) - } - - if size <= 0 { - t.Error("Database size should be greater than 0") - } - - // Verify size matches actual file size - stat, err := os.Stat(db.config.Path) - if err != nil { - t.Fatal("Failed to stat database file:", err) - } - - if size != stat.Size() { - t.Errorf("getDatabaseSize returned %d, but file size is %d", size, stat.Size()) - } -} - -func TestOptimizationManager_InvalidPath(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - // Test with invalid path - config := &Config{Path: "/nonexistent/path/database.db"} - optimizer := NewOptimizationManager(db, config) - - _, err := optimizer.getDatabaseSize() - if err == nil { - t.Error("getDatabaseSize should fail with invalid path") - } -} - -func TestOptimizationStats_JSON(t *testing.T) { - stats := &OptimizationStats{ - DatabaseSize: 1024000, - PageSize: 4096, - PageCount: 250, - UsedPages: 200, - FreePages: 50, - Efficiency: 80.0, - AutoVacuumEnabled: true, - LastVacuum: time.Now(), - } - - // Test that all fields are accessible - if stats.DatabaseSize != 1024000 { - t.Error("DatabaseSize not preserved") - } - if stats.PageSize != 4096 { - t.Error("PageSize not preserved") - } - if stats.Efficiency != 80.0 { - t.Error("Efficiency not preserved") - } - if !stats.AutoVacuumEnabled { - t.Error("AutoVacuumEnabled not preserved") - } -} - -func TestOptimizationManager_WithRealData(t *testing.T) { - db, cleanup := setupTestDatabase(t) - defer cleanup() - - // Load some real data to make optimization more realistic - // Skip actual data loading in tests as it requires network access - // Just insert minimal test data - conn := db.GetConnection() - _, err := conn.Exec(`INSERT INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source) - VALUES (1, 'Test Airways', 'Test', 'TA', 'TST', 'TESTAIR', 'United States', 1, 'test')`) - if err != nil { - t.Fatal("Failed to insert test data:", err) - } - - config := &Config{Path: db.config.Path} - optimizer := NewOptimizationManager(db, config) - - // Get stats before optimization - statsBefore, err := optimizer.GetOptimizationStats() - if err != nil { - t.Fatal("Failed to get stats before optimization:", err) - } - - // Run optimization - err = optimizer.OptimizeDatabase() - if err != nil { - t.Fatal("OptimizeDatabase failed:", err) - } - - err = optimizer.VacuumDatabase() - if err != nil { - t.Fatal("VacuumDatabase failed:", err) - } - - // Get stats after optimization - statsAfter, err := optimizer.GetOptimizationStats() - if err != nil { - t.Fatal("Failed to get stats after optimization:", err) - } - - // Compare efficiency - t.Logf("Optimization results: %.2f%% → %.2f%% efficiency", - statsBefore.Efficiency, statsAfter.Efficiency) - - // After optimization, we should have auto-vacuum enabled - if !statsAfter.AutoVacuumEnabled { - t.Error("Auto-vacuum should be enabled after optimization") - } - - // Database should still be functional - conn = db.GetConnection() - var count int - err = conn.QueryRow("SELECT COUNT(*) FROM airlines").Scan(&count) - if err != nil { - t.Error("Database not functional after optimization:", err) - } - if count == 0 { - t.Error("Data lost during optimization") - } -} \ No newline at end of file diff --git a/internal/database/test_helpers.go b/internal/database/test_helpers.go deleted file mode 100644 index 9d4bcee..0000000 --- a/internal/database/test_helpers.go +++ /dev/null @@ -1,36 +0,0 @@ -package database - -import ( - "os" - "testing" -) - -// setupTestDatabase creates a temporary database for testing -func setupTestDatabase(t *testing.T) (*Database, func()) { - tempFile, err := os.CreateTemp("", "test_skyview_*.db") - if err != nil { - t.Fatal("Failed to create temp database file:", err) - } - tempFile.Close() - - config := &Config{Path: tempFile.Name()} - db, err := NewDatabase(config) - if err != nil { - t.Fatal("Failed to create database:", err) - } - - // Initialize the database (run migrations) - err = db.Initialize() - if err != nil { - db.Close() - os.Remove(tempFile.Name()) - t.Fatal("Failed to initialize database:", err) - } - - cleanup := func() { - db.Close() - os.Remove(tempFile.Name()) - } - - return db, cleanup -} \ No newline at end of file