style: Apply code formatting with go fmt
- Run 'make format' to ensure all Go code follows standard formatting - Maintains consistent code style across the entire codebase - No functional changes, only whitespace and formatting improvements
This commit is contained in:
parent
4fd0846127
commit
2bffa2c418
19 changed files with 543 additions and 527 deletions
|
|
@ -37,32 +37,32 @@ type DataSource struct {
|
|||
|
||||
// LoadResult contains the results of a data loading operation
|
||||
type LoadResult struct {
|
||||
Source string `json:"source"`
|
||||
RecordsTotal int `json:"records_total"`
|
||||
RecordsNew int `json:"records_new"`
|
||||
RecordsError int `json:"records_error"`
|
||||
Source string `json:"source"`
|
||||
RecordsTotal int `json:"records_total"`
|
||||
RecordsNew int `json:"records_new"`
|
||||
RecordsError int `json:"records_error"`
|
||||
Duration time.Duration `json:"duration"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// NewDataLoader creates a new data loader with HTTP client
|
||||
func NewDataLoader(conn *sql.DB) *DataLoader {
|
||||
// Check for insecure TLS environment variable
|
||||
insecureTLS := os.Getenv("SKYVIEW_INSECURE_TLS") == "1"
|
||||
|
||||
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: false,
|
||||
}
|
||||
|
||||
|
||||
// Allow insecure certificates if requested
|
||||
if insecureTLS {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return &DataLoader{
|
||||
conn: conn,
|
||||
client: &http.Client{
|
||||
|
|
@ -85,7 +85,7 @@ func GetAvailableDataSources() []DataSource {
|
|||
},
|
||||
{
|
||||
Name: "OpenFlights Airports",
|
||||
License: "AGPL-3.0",
|
||||
License: "AGPL-3.0",
|
||||
URL: "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat",
|
||||
RequiresConsent: false, // Runtime data consumption doesn't require explicit consent
|
||||
Format: "openflights",
|
||||
|
|
@ -111,23 +111,23 @@ func (dl *DataLoader) LoadDataSource(source DataSource) (*LoadResult, error) {
|
|||
defer func() {
|
||||
result.Duration = time.Since(startTime)
|
||||
}()
|
||||
|
||||
|
||||
// Check license acceptance if required
|
||||
if source.RequiresConsent && !source.UserAcceptedLicense {
|
||||
return nil, fmt.Errorf("user has not accepted license for source: %s (%s)", source.Name, source.License)
|
||||
}
|
||||
|
||||
|
||||
// Download data
|
||||
resp, err := dl.client.Get(source.URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to download data from %s: %v", source.URL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP error downloading data: %s", resp.Status)
|
||||
}
|
||||
|
||||
|
||||
// Parse and load data based on format
|
||||
switch source.Format {
|
||||
case "openflights":
|
||||
|
|
@ -137,10 +137,10 @@ func (dl *DataLoader) LoadDataSource(source DataSource) (*LoadResult, error) {
|
|||
return dl.loadOpenFlightsAirports(resp.Body, source, result)
|
||||
}
|
||||
return nil, fmt.Errorf("unknown OpenFlights data type: %s", source.Name)
|
||||
|
||||
|
||||
case "ourairports":
|
||||
return dl.loadOurAirports(resp.Body, source, result)
|
||||
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported data format: %s", source.Format)
|
||||
}
|
||||
|
|
@ -153,21 +153,21 @@ func (dl *DataLoader) loadOpenFlightsAirlines(reader io.Reader, source DataSourc
|
|||
return nil, fmt.Errorf("failed to begin transaction: %v", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
|
||||
// Record data source
|
||||
if err := dl.recordDataSource(tx, source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
// Clear existing data from this source
|
||||
_, err = tx.Exec(`DELETE FROM airlines WHERE data_source = ?`, source.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to clear existing airline data: %v", err)
|
||||
}
|
||||
|
||||
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.FieldsPerRecord = -1 // Variable number of fields
|
||||
|
||||
|
||||
insertStmt, err := tx.Prepare(`
|
||||
INSERT OR REPLACE INTO airlines (id, name, alias, iata_code, icao_code, callsign, country, active, data_source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
|
|
@ -176,7 +176,7 @@ func (dl *DataLoader) loadOpenFlightsAirlines(reader io.Reader, source DataSourc
|
|||
return nil, fmt.Errorf("failed to prepare insert statement: %v", err)
|
||||
}
|
||||
defer insertStmt.Close()
|
||||
|
||||
|
||||
for {
|
||||
record, err := csvReader.Read()
|
||||
if err == io.EOF {
|
||||
|
|
@ -187,15 +187,15 @@ func (dl *DataLoader) loadOpenFlightsAirlines(reader io.Reader, source DataSourc
|
|||
result.Errors = append(result.Errors, fmt.Sprintf("CSV parse error: %v", err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
if len(record) < 7 {
|
||||
result.RecordsError++
|
||||
result.Errors = append(result.Errors, "insufficient fields in record")
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
result.RecordsTotal++
|
||||
|
||||
|
||||
// Parse OpenFlights airline format:
|
||||
// ID, Name, Alias, IATA, ICAO, Callsign, Country, Active
|
||||
id, _ := strconv.Atoi(record[0])
|
||||
|
|
@ -206,29 +206,37 @@ func (dl *DataLoader) loadOpenFlightsAirlines(reader io.Reader, source DataSourc
|
|||
callsign := strings.Trim(record[5], `"`)
|
||||
country := strings.Trim(record[6], `"`)
|
||||
active := len(record) > 7 && strings.Trim(record[7], `"`) == "Y"
|
||||
|
||||
|
||||
// Convert \N to empty strings
|
||||
if alias == "\\N" { alias = "" }
|
||||
if iata == "\\N" { iata = "" }
|
||||
if icao == "\\N" { icao = "" }
|
||||
if callsign == "\\N" { callsign = "" }
|
||||
|
||||
if alias == "\\N" {
|
||||
alias = ""
|
||||
}
|
||||
if iata == "\\N" {
|
||||
iata = ""
|
||||
}
|
||||
if icao == "\\N" {
|
||||
icao = ""
|
||||
}
|
||||
if callsign == "\\N" {
|
||||
callsign = ""
|
||||
}
|
||||
|
||||
_, err = insertStmt.Exec(id, name, alias, iata, icao, callsign, country, active, source.Name)
|
||||
if err != nil {
|
||||
result.RecordsError++
|
||||
result.Errors = append(result.Errors, fmt.Sprintf("insert error for airline %s: %v", name, err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
result.RecordsNew++
|
||||
}
|
||||
|
||||
|
||||
// Update record count
|
||||
_, err = tx.Exec(`UPDATE data_sources SET record_count = ? WHERE name = ?`, result.RecordsNew, source.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to update record count: %v", err)
|
||||
}
|
||||
|
||||
|
||||
return result, tx.Commit()
|
||||
}
|
||||
|
||||
|
|
@ -239,21 +247,21 @@ func (dl *DataLoader) loadOpenFlightsAirports(reader io.Reader, source DataSourc
|
|||
return nil, fmt.Errorf("failed to begin transaction: %v", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
|
||||
// Record data source
|
||||
if err := dl.recordDataSource(tx, source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
// Clear existing data from this source
|
||||
_, err = tx.Exec(`DELETE FROM airports WHERE data_source = ?`, source.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to clear existing airport data: %v", err)
|
||||
}
|
||||
|
||||
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.FieldsPerRecord = -1
|
||||
|
||||
|
||||
insertStmt, err := tx.Prepare(`
|
||||
INSERT OR REPLACE INTO airports (id, name, city, country, iata_code, icao_code, latitude, longitude,
|
||||
elevation_ft, timezone_offset, dst_type, timezone, data_source)
|
||||
|
|
@ -263,7 +271,7 @@ func (dl *DataLoader) loadOpenFlightsAirports(reader io.Reader, source DataSourc
|
|||
return nil, fmt.Errorf("failed to prepare insert statement: %v", err)
|
||||
}
|
||||
defer insertStmt.Close()
|
||||
|
||||
|
||||
for {
|
||||
record, err := csvReader.Read()
|
||||
if err == io.EOF {
|
||||
|
|
@ -274,15 +282,15 @@ func (dl *DataLoader) loadOpenFlightsAirports(reader io.Reader, source DataSourc
|
|||
result.Errors = append(result.Errors, fmt.Sprintf("CSV parse error: %v", err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
if len(record) < 12 {
|
||||
result.RecordsError++
|
||||
result.Errors = append(result.Errors, "insufficient fields in airport record")
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
result.RecordsTotal++
|
||||
|
||||
|
||||
// Parse OpenFlights airport format
|
||||
id, _ := strconv.Atoi(record[0])
|
||||
name := strings.Trim(record[1], `"`)
|
||||
|
|
@ -296,29 +304,37 @@ func (dl *DataLoader) loadOpenFlightsAirports(reader io.Reader, source DataSourc
|
|||
tzOffset, _ := strconv.ParseFloat(record[9], 64)
|
||||
dst := strings.Trim(record[10], `"`)
|
||||
timezone := strings.Trim(record[11], `"`)
|
||||
|
||||
|
||||
// Convert \N to empty strings
|
||||
if iata == "\\N" { iata = "" }
|
||||
if icao == "\\N" { icao = "" }
|
||||
if dst == "\\N" { dst = "" }
|
||||
if timezone == "\\N" { timezone = "" }
|
||||
|
||||
if iata == "\\N" {
|
||||
iata = ""
|
||||
}
|
||||
if icao == "\\N" {
|
||||
icao = ""
|
||||
}
|
||||
if dst == "\\N" {
|
||||
dst = ""
|
||||
}
|
||||
if timezone == "\\N" {
|
||||
timezone = ""
|
||||
}
|
||||
|
||||
_, err = insertStmt.Exec(id, name, city, country, iata, icao, lat, lon, alt, tzOffset, dst, timezone, source.Name)
|
||||
if err != nil {
|
||||
result.RecordsError++
|
||||
result.Errors = append(result.Errors, fmt.Sprintf("insert error for airport %s: %v", name, err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
result.RecordsNew++
|
||||
}
|
||||
|
||||
|
||||
// Update record count
|
||||
_, err = tx.Exec(`UPDATE data_sources SET record_count = ? WHERE name = ?`, result.RecordsNew, source.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to update record count: %v", err)
|
||||
}
|
||||
|
||||
|
||||
return result, tx.Commit()
|
||||
}
|
||||
|
||||
|
|
@ -330,9 +346,9 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
return nil, fmt.Errorf("failed to begin transaction: %v", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
|
||||
csvReader := csv.NewReader(reader)
|
||||
|
||||
|
||||
// Read header row
|
||||
headers, err := csvReader.Read()
|
||||
if err != nil {
|
||||
|
|
@ -340,13 +356,13 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
result.Errors = []string{fmt.Sprintf("Failed to read CSV header: %v", err)}
|
||||
return result, err
|
||||
}
|
||||
|
||||
|
||||
// Create header index map for easier field access
|
||||
headerIndex := make(map[string]int)
|
||||
for i, header := range headers {
|
||||
headerIndex[strings.TrimSpace(header)] = i
|
||||
}
|
||||
|
||||
|
||||
// Prepare statement for airports
|
||||
stmt, err := tx.Prepare(`
|
||||
INSERT OR REPLACE INTO airports (
|
||||
|
|
@ -362,7 +378,7 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
return result, err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
|
||||
// Process each row
|
||||
for {
|
||||
record, err := csvReader.Read()
|
||||
|
|
@ -374,13 +390,13 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
result.Errors = append(result.Errors, fmt.Sprintf("CSV read error: %v", err))
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
// Skip rows with insufficient fields
|
||||
if len(record) < len(headerIndex) {
|
||||
result.RecordsError++
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
// Extract fields using header index
|
||||
sourceID := getFieldByHeader(record, headerIndex, "id")
|
||||
ident := getFieldByHeader(record, headerIndex, "ident")
|
||||
|
|
@ -394,7 +410,7 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
homeLink := getFieldByHeader(record, headerIndex, "home_link")
|
||||
wikipediaLink := getFieldByHeader(record, headerIndex, "wikipedia_link")
|
||||
keywords := getFieldByHeader(record, headerIndex, "keywords")
|
||||
|
||||
|
||||
// Parse coordinates
|
||||
var latitude, longitude float64
|
||||
if latStr := getFieldByHeader(record, headerIndex, "latitude_deg"); latStr != "" {
|
||||
|
|
@ -407,7 +423,7 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
longitude = lng
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Parse elevation
|
||||
var elevation int
|
||||
if elevStr := getFieldByHeader(record, headerIndex, "elevation_ft"); elevStr != "" {
|
||||
|
|
@ -415,10 +431,10 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
elevation = elev
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Parse scheduled service
|
||||
scheduledService := getFieldByHeader(record, headerIndex, "scheduled_service") == "yes"
|
||||
|
||||
|
||||
// Insert airport record
|
||||
_, err = stmt.Exec(
|
||||
sourceID, name, ident, airportType, icaoCode, iataCode,
|
||||
|
|
@ -432,7 +448,7 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
result.RecordsNew++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Update data source tracking
|
||||
_, err = tx.Exec(`
|
||||
INSERT OR REPLACE INTO data_sources (name, license, url, imported_at, record_count, user_accepted_license)
|
||||
|
|
@ -441,7 +457,7 @@ func (dl *DataLoader) loadOurAirports(reader io.Reader, source DataSource, resul
|
|||
if err != nil {
|
||||
return result, fmt.Errorf("failed to update data source tracking: %v", err)
|
||||
}
|
||||
|
||||
|
||||
return result, tx.Commit()
|
||||
}
|
||||
|
||||
|
|
@ -460,13 +476,13 @@ func (dl *DataLoader) GetLoadedDataSources() ([]DataSource, error) {
|
|||
FROM data_sources
|
||||
ORDER BY name
|
||||
`
|
||||
|
||||
|
||||
rows, err := dl.conn.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
|
||||
var sources []DataSource
|
||||
for rows.Next() {
|
||||
var source DataSource
|
||||
|
|
@ -482,7 +498,7 @@ func (dl *DataLoader) GetLoadedDataSources() ([]DataSource, error) {
|
|||
}
|
||||
sources = append(sources, source)
|
||||
}
|
||||
|
||||
|
||||
return sources, rows.Err()
|
||||
}
|
||||
|
||||
|
|
@ -493,11 +509,10 @@ func (dl *DataLoader) recordDataSource(tx *sql.Tx, source DataSource) error {
|
|||
(name, license, url, version, user_accepted_license)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, source.Name, source.License, source.URL, source.Version, source.UserAcceptedLicense)
|
||||
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
// ClearDataSource removes all data from a specific source
|
||||
func (dl *DataLoader) ClearDataSource(sourceName string) error {
|
||||
tx, err := dl.conn.Begin()
|
||||
|
|
@ -505,22 +520,22 @@ func (dl *DataLoader) ClearDataSource(sourceName string) error {
|
|||
return fmt.Errorf("failed to begin transaction: %v", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
|
||||
// Clear from all tables
|
||||
_, err = tx.Exec(`DELETE FROM airlines WHERE data_source = ?`, sourceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to clear airlines: %v", err)
|
||||
}
|
||||
|
||||
|
||||
_, err = tx.Exec(`DELETE FROM airports WHERE data_source = ?`, sourceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to clear airports: %v", err)
|
||||
}
|
||||
|
||||
|
||||
_, err = tx.Exec(`DELETE FROM data_sources WHERE name = ?`, sourceName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to clear data source record: %v", err)
|
||||
}
|
||||
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue