fitpub/src/main/java/org/operaton/fitpub/util/FitParser.java
Tim Zöller f4be439002 Add GPX file support for activity imports
This commit adds comprehensive GPX file support alongside existing FIT file support, enabling users to import activities from Strava, Komoot, and other GPS apps.

## Key Features

### Core Components
- **GpxParser**: Full GPX 1.1 parsing with Garmin TrackPointExtension support
- **GpxFileValidator**: Validation for GPX file format and structure
- **ActivityFileService**: Unified service with automatic format detection (FIT/GPX)
- **ParsedActivityData**: Common data structure for both FIT and GPX files

### GPX Parsing Capabilities
- GPS track point extraction (latitude, longitude, elevation, timestamp)
- Garmin extension data (heart rate, cadence, temperature)
- Activity type detection from GPX metadata
- Distance calculation using Haversine formula
- Elevation gain/loss calculation
- Speed calculation from consecutive GPS points
- Speed smoothing to remove GPS artifacts
- Timezone detection from GPS coordinates
- Moving time vs. stopped time analysis

### Database Changes
- Migration V15: Renamed raw_fit_file → raw_activity_file
- Added source_file_format column (FIT/GPX) with constraint
- Index on source_file_format for performance
- Updated Activity entity with new fields

### Controller & UI Updates
- ActivityController: Now handles both FIT and GPX uploads
- Upload form: Updated to accept .fit and .gpx files
- Help text: Clarified both formats are supported

### Testing
- GpxParserIntegrationTest: 9 comprehensive tests with real GPX file
- Tests cover: parsing, validation, heart rate extraction, distance calculation,
  elevation metrics, speed calculation, chronological ordering, smoothing
- Fixed TrainingLoadServiceTest date issue (testDate outside 30-day window)
- All 97 unit tests passing (integration tests require Docker)

### Technical Details
- Supports GPX 1.0 and 1.1 specifications
- Handles multiple track segments
- Processes Garmin TrackPointExtension v1 and v2
- Same track simplification as FIT (Douglas-Peucker algorithm)
- Consistent JSONB storage format for track points
- Compatible with existing analytics, heatmaps, and image generation

## Testing Summary
-  9/9 GpxParserIntegrationTest tests passing
-  4/4 FitParserIntegrationTest tests passing
-  14/14 FitFileServiceTest tests passing
-  97/97 total unit tests passing

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-01-02 13:31:05 +01:00

401 lines
14 KiB
Java

package org.operaton.fitpub.util;
import com.garmin.fit.*;
import lombok.extern.slf4j.Slf4j;
import net.iakovlev.timeshape.TimeZoneEngine;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.PrecisionModel;
import org.operaton.fitpub.exception.FitFileProcessingException;
import org.operaton.fitpub.model.entity.Activity;
import org.operaton.fitpub.model.entity.ActivityMetrics;
import org.operaton.fitpub.util.ParsedActivityData.ActivityMetricsData;
import org.operaton.fitpub.util.ParsedActivityData.TrackPointData;
import org.springframework.stereotype.Component;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/**
* Parser for Garmin FIT files.
* Extracts GPS coordinates, activity metrics, and sensor data.
*/
@Component
@Slf4j
public class FitParser {
private static final int WGS84_SRID = 4326;
private static final GeometryFactory GEOMETRY_FACTORY =
new GeometryFactory(new PrecisionModel(), WGS84_SRID);
private static final double SEMICIRCLES_TO_DEGREES = 180.0 / Math.pow(2, 31);
private static final double MPS_TO_KPH = 3.6;
// Lazy-loaded timezone engine (expensive to initialize)
private static TimeZoneEngine timezoneEngine = null;
private final SpeedSmoother speedSmoother;
public FitParser(SpeedSmoother speedSmoother) {
this.speedSmoother = speedSmoother;
}
/**
* Parses a FIT file and returns the extracted data.
*
* @param fileData the FIT file data
* @return ParsedActivityData containing activity information
* @throws FitFileProcessingException if parsing fails
*/
public ParsedActivityData parse(byte[] fileData) {
try (InputStream inputStream = new ByteArrayInputStream(fileData)) {
return parse(inputStream);
} catch (Exception e) {
throw new FitFileProcessingException("Failed to parse FIT file", e);
}
}
/**
* Parses a FIT file from an input stream.
*
* @param inputStream the input stream
* @return ParsedActivityData containing activity information
* @throws FitFileProcessingException if parsing fails
*/
public ParsedActivityData parse(InputStream inputStream) {
ParsedActivityData parsedData = new ParsedActivityData();
Decode decode = new Decode();
MesgBroadcaster broadcaster = new MesgBroadcaster(decode);
// Listen for record messages (GPS points)
broadcaster.addListener((RecordMesgListener) record -> {
TrackPointData trackPoint = extractTrackPoint(record);
if (trackPoint != null) {
parsedData.getTrackPoints().add(trackPoint);
}
});
// Listen for session messages (summary data)
broadcaster.addListener((SessionMesgListener) session -> {
extractSessionData(session, parsedData);
});
// Listen for activity messages
broadcaster.addListener((ActivityMesgListener) activity -> {
extractActivityData(activity, parsedData);
});
// Listen for lap messages
broadcaster.addListener((LapMesgListener) lap -> {
log.debug("Lap data: distance={}, time={}", lap.getTotalDistance(), lap.getTotalTimerTime());
});
try {
if (!decode.read(inputStream, broadcaster)) {
throw new FitFileProcessingException("Failed to decode FIT file");
}
if (parsedData.getTrackPoints().isEmpty()) {
throw new FitFileProcessingException("No GPS track points found in FIT file");
}
// Determine timezone from first GPS coordinate
determineTimezone(parsedData);
// Apply speed smoothing and recalculate max speed
smoothSpeedData(parsedData);
log.info("Successfully parsed FIT file: {} track points, activity type: {}, timezone: {}",
parsedData.getTrackPoints().size(), parsedData.getActivityType(), parsedData.getTimezone());
return parsedData;
} catch (FitRuntimeException e) {
throw new FitFileProcessingException("Error decoding FIT file", e);
}
}
/**
* Extracts a track point from a record message.
*/
private TrackPointData extractTrackPoint(RecordMesg record) {
Integer positionLat = record.getPositionLat();
Integer positionLong = record.getPositionLong();
if (positionLat == null || positionLong == null) {
return null; // Skip points without GPS coordinates
}
TrackPointData point = new TrackPointData();
// Convert semicircles to degrees
double latitude = positionLat * SEMICIRCLES_TO_DEGREES;
double longitude = positionLong * SEMICIRCLES_TO_DEGREES;
point.setLatitude(latitude);
point.setLongitude(longitude);
// Extract timestamp
if (record.getTimestamp() != null) {
point.setTimestamp(convertDateTime(record.getTimestamp()));
}
// Extract elevation
if (record.getAltitude() != null) {
point.setElevation(BigDecimal.valueOf(record.getAltitude()).setScale(2, RoundingMode.HALF_UP));
}
// Extract heart rate
if (record.getHeartRate() != null) {
point.setHeartRate(record.getHeartRate().intValue());
}
// Extract cadence
if (record.getCadence() != null) {
point.setCadence(record.getCadence().intValue());
}
// Extract power
if (record.getPower() != null) {
point.setPower(record.getPower());
}
// Extract speed (convert m/s to km/h)
if (record.getSpeed() != null) {
point.setSpeed(BigDecimal.valueOf(record.getSpeed() * MPS_TO_KPH)
.setScale(2, RoundingMode.HALF_UP));
}
// Extract distance
if (record.getDistance() != null) {
point.setDistance(BigDecimal.valueOf(record.getDistance()).setScale(2, RoundingMode.HALF_UP));
}
// Extract temperature
if (record.getTemperature() != null) {
point.setTemperature(BigDecimal.valueOf(record.getTemperature()).setScale(2, RoundingMode.HALF_UP));
}
return point;
}
/**
* Extracts session data from a session message.
*/
private void extractSessionData(SessionMesg session, ParsedActivityData parsedData) {
if (session.getStartTime() != null) {
parsedData.setStartTime(convertDateTime(session.getStartTime()));
}
if (session.getTotalElapsedTime() != null) {
long totalSeconds = session.getTotalElapsedTime().longValue();
parsedData.setTotalDuration(Duration.ofSeconds(totalSeconds));
if (parsedData.getStartTime() != null) {
parsedData.setEndTime(parsedData.getStartTime().plus(Duration.ofSeconds(totalSeconds)));
}
}
if (session.getTotalDistance() != null) {
parsedData.setTotalDistance(
BigDecimal.valueOf(session.getTotalDistance()).setScale(2, RoundingMode.HALF_UP)
);
}
if (session.getTotalAscent() != null) {
parsedData.setElevationGain(
BigDecimal.valueOf(session.getTotalAscent()).setScale(2, RoundingMode.HALF_UP)
);
}
if (session.getTotalDescent() != null) {
parsedData.setElevationLoss(
BigDecimal.valueOf(session.getTotalDescent()).setScale(2, RoundingMode.HALF_UP)
);
}
// Extract metrics
ActivityMetricsData metrics = new ActivityMetricsData();
if (session.getAvgSpeed() != null) {
metrics.setAverageSpeed(
BigDecimal.valueOf(session.getAvgSpeed() * MPS_TO_KPH).setScale(2, RoundingMode.HALF_UP)
);
}
if (session.getMaxSpeed() != null) {
metrics.setMaxSpeed(
BigDecimal.valueOf(session.getMaxSpeed() * MPS_TO_KPH).setScale(2, RoundingMode.HALF_UP)
);
}
if (session.getAvgHeartRate() != null) {
metrics.setAverageHeartRate(session.getAvgHeartRate().intValue());
}
if (session.getMaxHeartRate() != null) {
metrics.setMaxHeartRate(session.getMaxHeartRate().intValue());
}
if (session.getAvgCadence() != null) {
metrics.setAverageCadence(session.getAvgCadence().intValue());
}
if (session.getMaxCadence() != null) {
metrics.setMaxCadence(session.getMaxCadence().intValue());
}
if (session.getAvgPower() != null) {
metrics.setAveragePower(session.getAvgPower());
}
if (session.getMaxPower() != null) {
metrics.setMaxPower(session.getMaxPower());
}
if (session.getNormalizedPower() != null) {
metrics.setNormalizedPower(session.getNormalizedPower());
}
if (session.getTotalCalories() != null) {
metrics.setCalories(session.getTotalCalories());
}
if (session.getTotalMovingTime() != null) {
metrics.setMovingTime(Duration.ofSeconds(session.getTotalMovingTime().longValue()));
}
if (session.getTotalStrides() != null) {
metrics.setTotalSteps(session.getTotalStrides().intValue() * 2); // Strides to steps
}
parsedData.setMetrics(metrics);
// Determine activity type
if (session.getSport() != null) {
parsedData.setActivityType(mapSportToActivityType(session.getSport()));
}
}
/**
* Extracts activity data from an activity message.
*/
private void extractActivityData(ActivityMesg activity, ParsedActivityData parsedData) {
if (activity.getTimestamp() != null) {
parsedData.setActivityTimestamp(convertDateTime(activity.getTimestamp()));
}
if (activity.getTotalTimerTime() != null) {
log.debug("Activity total timer time: {}", activity.getTotalTimerTime());
}
}
/**
* Applies speed smoothing to track points and updates max speed in metrics.
* Removes unrealistic GPS speed spikes and recalculates max speed.
*/
private void smoothSpeedData(ParsedActivityData parsedData) {
if (parsedData.getTrackPoints().isEmpty() || parsedData.getMetrics() == null) {
return;
}
// Smooth speed data and get recalculated max speed
BigDecimal smoothedMaxSpeed = speedSmoother.smoothAndCalculateMaxSpeed(
parsedData.getTrackPoints(),
parsedData.getActivityType()
);
// Update max speed in metrics if we got a valid smoothed value
if (smoothedMaxSpeed != null) {
BigDecimal originalMaxSpeed = parsedData.getMetrics().getMaxSpeed();
parsedData.getMetrics().setMaxSpeed(smoothedMaxSpeed);
if (originalMaxSpeed != null && smoothedMaxSpeed.compareTo(originalMaxSpeed) < 0) {
log.info("Smoothed max speed from {} km/h to {} km/h (removed GPS artifacts)",
originalMaxSpeed, smoothedMaxSpeed);
}
}
}
/**
* Determines the timezone based on the first GPS coordinate.
* Uses TimeZoneEngine library for accurate timezone lookup from coordinates.
*/
private void determineTimezone(ParsedActivityData parsedData) {
if (parsedData.getTrackPoints().isEmpty()) {
parsedData.setTimezone("UTC");
return;
}
TrackPointData firstPoint = parsedData.getTrackPoints().get(0);
double latitude = firstPoint.getLatitude();
double longitude = firstPoint.getLongitude();
try {
// Lazy-load timezone engine (expensive initialization ~200ms first time)
if (timezoneEngine == null) {
log.info("Initializing TimeZoneEngine for timezone lookup...");
timezoneEngine = TimeZoneEngine.initialize();
}
Optional<ZoneId> zoneId = timezoneEngine.query(latitude, longitude);
if (zoneId.isPresent()) {
parsedData.setTimezone(zoneId.get().getId());
log.debug("Determined timezone: {} from coordinates ({}, {})",
zoneId.get().getId(), latitude, longitude);
} else {
log.warn("Could not determine timezone for coordinates ({}, {}), defaulting to UTC",
latitude, longitude);
parsedData.setTimezone("UTC");
}
} catch (Exception e) {
log.error("Error determining timezone, defaulting to UTC", e);
parsedData.setTimezone("UTC");
}
}
/**
* Converts FIT DateTime to LocalDateTime.
* FIT timestamps use a special epoch: December 31, 1989, 00:00:00 UTC.
* We need to add the offset from Unix epoch (1970) to FIT epoch (1989).
*/
private LocalDateTime convertDateTime(DateTime dateTime) {
// FIT epoch offset: seconds between 1970-01-01 and 1989-12-31
final long FIT_EPOCH_OFFSET = 631065600L;
long timestamp = dateTime.getTimestamp();
// Add FIT epoch offset to convert to Unix timestamp
Instant instant = Instant.ofEpochSecond(timestamp + FIT_EPOCH_OFFSET);
return LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
}
/**
* Maps FIT sport type to our activity type.
*/
private Activity.ActivityType mapSportToActivityType(Sport sport) {
if (sport == Sport.RUNNING) {
return Activity.ActivityType.RUN;
} else if (sport == Sport.CYCLING) {
return Activity.ActivityType.RIDE;
} else if (sport == Sport.HIKING) {
return Activity.ActivityType.HIKE;
} else if (sport == Sport.WALKING) {
return Activity.ActivityType.WALK;
} else if (sport == Sport.SWIMMING) {
return Activity.ActivityType.SWIM;
} else if (sport == Sport.ROWING) {
return Activity.ActivityType.ROWING;
} else {
return Activity.ActivityType.OTHER;
}
}
}