Improvements
This commit is contained in:
parent
139e5c57ff
commit
164e8b5894
8 changed files with 210 additions and 11 deletions
|
|
@ -28,6 +28,8 @@ public class DebugController {
|
||||||
|
|
||||||
private final UserRepository userRepository;
|
private final UserRepository userRepository;
|
||||||
private final net.javahippie.fitpub.service.PeakDetectionService peakDetectionService;
|
private final net.javahippie.fitpub.service.PeakDetectionService peakDetectionService;
|
||||||
|
private final net.javahippie.fitpub.service.ActivityFileService activityFileService;
|
||||||
|
private final net.javahippie.fitpub.repository.ActivityRepository activityRepository;
|
||||||
|
|
||||||
@GetMapping("/validate-keys")
|
@GetMapping("/validate-keys")
|
||||||
public Map<String, Object> validateKeys() {
|
public Map<String, Object> validateKeys() {
|
||||||
|
|
@ -117,4 +119,49 @@ public class DebugController {
|
||||||
peakDetectionService.backfillAllActivities();
|
peakDetectionService.backfillAllActivities();
|
||||||
return org.springframework.http.ResponseEntity.ok(Map.of("status", "started"));
|
return org.springframework.http.ResponseEntity.ok(Map.of("status", "started"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private final java.util.concurrent.atomic.AtomicBoolean elevationReprocessRunning =
|
||||||
|
new java.util.concurrent.atomic.AtomicBoolean(false);
|
||||||
|
|
||||||
|
@org.springframework.web.bind.annotation.PostMapping("/reprocess-gpx-elevation")
|
||||||
|
public org.springframework.http.ResponseEntity<Map<String, String>> reprocessGpxElevation() {
|
||||||
|
if (!elevationReprocessRunning.compareAndSet(false, true)) {
|
||||||
|
return org.springframework.http.ResponseEntity.ok(Map.of("status", "already running"));
|
||||||
|
}
|
||||||
|
|
||||||
|
new Thread(() -> {
|
||||||
|
try {
|
||||||
|
var ids = activityRepository.findAllIds();
|
||||||
|
log.info("Starting GPX elevation reprocessing for {} activities", ids.size());
|
||||||
|
|
||||||
|
int updated = 0, skipped = 0, failed = 0;
|
||||||
|
for (int i = 0; i < ids.size(); i++) {
|
||||||
|
try {
|
||||||
|
var activity = activityRepository.findById(ids.get(i)).orElse(null);
|
||||||
|
if (activity == null || !"GPX".equals(activity.getSourceFileFormat())) {
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (activityFileService.reprocessGpxElevation(activity)) {
|
||||||
|
updated++;
|
||||||
|
} else {
|
||||||
|
skipped++;
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
failed++;
|
||||||
|
log.warn("Failed to reprocess elevation for {}: {}", ids.get(i), e.getMessage());
|
||||||
|
}
|
||||||
|
if ((i + 1) % 100 == 0) {
|
||||||
|
log.info("GPX elevation reprocess progress: {} / {} (updated={}, skipped={}, failed={})",
|
||||||
|
i + 1, ids.size(), updated, skipped, failed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info("GPX elevation reprocessing complete: updated={}, skipped={}, failed={}", updated, skipped, failed);
|
||||||
|
} finally {
|
||||||
|
elevationReprocessRunning.set(false);
|
||||||
|
}
|
||||||
|
}, "gpx-elevation-reprocess").start();
|
||||||
|
|
||||||
|
return org.springframework.http.ResponseEntity.ok(Map.of("status", "started"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,7 @@ public interface ActivityPeakRepository extends JpaRepository<ActivityPeak, Inte
|
||||||
JOIN peaks p ON p.id = ap.peak_id
|
JOIN peaks p ON p.id = ap.peak_id
|
||||||
JOIN activities a ON a.id = ap.activity_id
|
JOIN activities a ON a.id = ap.activity_id
|
||||||
WHERE a.user_id = :userId
|
WHERE a.user_id = :userId
|
||||||
|
AND a.visibility = 'PUBLIC'
|
||||||
GROUP BY p.id, p.name, p.wikipedia
|
GROUP BY p.id, p.name, p.wikipedia
|
||||||
ORDER BY p.name
|
ORDER BY p.name
|
||||||
""",
|
""",
|
||||||
|
|
|
||||||
|
|
@ -13,12 +13,15 @@ public interface PeakRepository extends JpaRepository<Peak, Integer> {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Find all peaks within a given distance (meters) of an activity's simplified track.
|
* Find all peaks within a given distance (meters) of an activity's simplified track.
|
||||||
* Uses ST_DWithin on geography type for accurate meter-based distance.
|
* Uses a two-stage approach:
|
||||||
|
* 1. ST_Expand + && for fast GIST index lookup on geometry
|
||||||
|
* 2. ST_DWithin on geography for precise meter-based filtering
|
||||||
*/
|
*/
|
||||||
@Query(value = """
|
@Query(value = """
|
||||||
SELECT p.* FROM peaks p
|
SELECT p.* FROM peaks p
|
||||||
JOIN activities a ON a.id = :activityId
|
JOIN activities a ON a.id = :activityId
|
||||||
WHERE a.simplified_track IS NOT NULL
|
WHERE a.simplified_track IS NOT NULL
|
||||||
|
AND p.geom && ST_Expand(a.simplified_track, 0.002)
|
||||||
AND ST_DWithin(CAST(p.geom AS geography), CAST(a.simplified_track AS geography), :distanceMeters)
|
AND ST_DWithin(CAST(p.geom AS geography), CAST(a.simplified_track AS geography), :distanceMeters)
|
||||||
ORDER BY p.name
|
ORDER BY p.name
|
||||||
""",
|
""",
|
||||||
|
|
|
||||||
|
|
@ -196,6 +196,42 @@ public class ActivityFileService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reprocess elevation for a GPX activity from its stored raw file.
|
||||||
|
* Only works for GPX files — FIT files use their session summary.
|
||||||
|
*
|
||||||
|
* @param activity the activity to reprocess
|
||||||
|
* @return true if elevation was updated
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public boolean reprocessGpxElevation(Activity activity) {
|
||||||
|
if (!"GPX".equals(activity.getSourceFileFormat())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[] rawFile = activity.getRawActivityFile();
|
||||||
|
if (rawFile == null || rawFile.length == 0) {
|
||||||
|
log.debug("No raw file stored for activity {}, skipping", activity.getId());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
ParsedActivityData parsedData = gpxParser.parse(rawFile);
|
||||||
|
|
||||||
|
BigDecimal oldGain = activity.getElevationGain();
|
||||||
|
activity.setElevationGain(parsedData.getElevationGain());
|
||||||
|
activity.setElevationLoss(parsedData.getElevationLoss());
|
||||||
|
activityRepository.save(activity);
|
||||||
|
|
||||||
|
log.info("Reprocessed GPX elevation for activity {}: {}m -> {}m",
|
||||||
|
activity.getId(), oldGain, parsedData.getElevationGain());
|
||||||
|
return true;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("Failed to reprocess elevation for activity {}: {}", activity.getId(), e.getMessage());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Detects file format from content and filename.
|
* Detects file format from content and filename.
|
||||||
* Priority: magic bytes > XML header > file extension
|
* Priority: magic bytes > XML header > file extension
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,7 @@ public class BatchImportService {
|
||||||
private final BatchImportJobRepository batchImportJobRepository;
|
private final BatchImportJobRepository batchImportJobRepository;
|
||||||
private final BatchImportFileResultRepository batchImportFileResultRepository;
|
private final BatchImportFileResultRepository batchImportFileResultRepository;
|
||||||
private final ActivityFileService activityFileService;
|
private final ActivityFileService activityFileService;
|
||||||
|
private final PeakDetectionService peakDetectionService;
|
||||||
private final ActivityRepository activityRepository;
|
private final ActivityRepository activityRepository;
|
||||||
private final UserRepository userRepository;
|
private final UserRepository userRepository;
|
||||||
private final PersonalRecordService personalRecordService;
|
private final PersonalRecordService personalRecordService;
|
||||||
|
|
@ -63,6 +64,7 @@ public class BatchImportService {
|
||||||
HeatmapGridService heatmapGridService,
|
HeatmapGridService heatmapGridService,
|
||||||
TrainingLoadService trainingLoadService,
|
TrainingLoadService trainingLoadService,
|
||||||
ActivitySummaryService activitySummaryService,
|
ActivitySummaryService activitySummaryService,
|
||||||
|
PeakDetectionService peakDetectionService,
|
||||||
@org.springframework.context.annotation.Lazy BatchImportService self) {
|
@org.springframework.context.annotation.Lazy BatchImportService self) {
|
||||||
this.batchImportJobRepository = batchImportJobRepository;
|
this.batchImportJobRepository = batchImportJobRepository;
|
||||||
this.batchImportFileResultRepository = batchImportFileResultRepository;
|
this.batchImportFileResultRepository = batchImportFileResultRepository;
|
||||||
|
|
@ -74,6 +76,7 @@ public class BatchImportService {
|
||||||
this.heatmapGridService = heatmapGridService;
|
this.heatmapGridService = heatmapGridService;
|
||||||
this.trainingLoadService = trainingLoadService;
|
this.trainingLoadService = trainingLoadService;
|
||||||
this.activitySummaryService = activitySummaryService;
|
this.activitySummaryService = activitySummaryService;
|
||||||
|
this.peakDetectionService = peakDetectionService;
|
||||||
this.self = self;
|
this.self = self;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -258,6 +261,13 @@ public class BatchImportService {
|
||||||
ActivityFileService.ProcessingOptions.batchImportMode()
|
ActivityFileService.ProcessingOptions.batchImportMode()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Detect peaks (now fast thanks to GIST index hit)
|
||||||
|
try {
|
||||||
|
peakDetectionService.detectPeaksForActivity(activity);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("Peak detection failed for activity {}: {}", activity.getId(), e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
// Mark file as success
|
// Mark file as success
|
||||||
fileResult.setStatus(BatchImportFileResult.FileStatus.SUCCESS);
|
fileResult.setStatus(BatchImportFileResult.FileStatus.SUCCESS);
|
||||||
fileResult.setActivityId(activity.getId());
|
fileResult.setActivityId(activity.getId());
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ public class GpxParser {
|
||||||
|
|
||||||
private static final String GPX_NS = "http://www.topografix.com/GPX/1/1";
|
private static final String GPX_NS = "http://www.topografix.com/GPX/1/1";
|
||||||
private static final String GPXTPX_NS = "http://www.garmin.com/xmlschemas/TrackPointExtension/v1";
|
private static final String GPXTPX_NS = "http://www.garmin.com/xmlschemas/TrackPointExtension/v1";
|
||||||
private static final double ELEVATION_NOISE_THRESHOLD = 2.0; // Ignore elevation changes < 2m
|
private static final double ELEVATION_NOISE_THRESHOLD = 0.0; // Sum all elevation changes (barometric data is smooth enough)
|
||||||
private static final double STOPPED_SPEED_THRESHOLD = 0.5; // km/h - below this is considered stopped
|
private static final double STOPPED_SPEED_THRESHOLD = 0.5; // km/h - below this is considered stopped
|
||||||
private static final long STOPPED_TIME_THRESHOLD = 30; // seconds - must be stopped this long to count
|
private static final long STOPPED_TIME_THRESHOLD = 30; // seconds - must be stopped this long to count
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -125,15 +125,6 @@ class DatePersistenceTest {
|
||||||
assertEquals(expectedEndTime, queriedActivity.getEndedAt(),
|
assertEquals(expectedEndTime, queriedActivity.getEndedAt(),
|
||||||
"Queried end time should match original parsed value");
|
"Queried end time should match original parsed value");
|
||||||
|
|
||||||
// Also verify it's recent (within last 2 months)
|
|
||||||
LocalDateTime now = LocalDateTime.now();
|
|
||||||
LocalDateTime twoMonthsAgo = now.minusMonths(2);
|
|
||||||
|
|
||||||
assertTrue(queriedActivity.getStartedAt().isAfter(twoMonthsAgo),
|
|
||||||
String.format("Activity should be recent. Expected after %s, got %s",
|
|
||||||
twoMonthsAgo.format(FORMATTER),
|
|
||||||
queriedActivity.getStartedAt().format(FORMATTER)));
|
|
||||||
|
|
||||||
log.info("");
|
log.info("");
|
||||||
log.info("✅ FIT file date persistence: PASSED");
|
log.info("✅ FIT file date persistence: PASSED");
|
||||||
log.info(" Expected: {}", expectedStartTime);
|
log.info(" Expected: {}", expectedStartTime);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,111 @@
|
||||||
|
package net.javahippie.fitpub.util;
|
||||||
|
|
||||||
|
import com.garmin.fit.*;
|
||||||
|
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.RoundingMode;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Standalone analysis of a FIT file's elevation data.
|
||||||
|
* Compares session summary vs track point calculation.
|
||||||
|
*/
|
||||||
|
public class FitElevationAnalysis {
|
||||||
|
|
||||||
|
@org.junit.jupiter.api.Test
|
||||||
|
public void analyzeElevation() {
|
||||||
|
analyze(new String[]{});
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void analyze(String[] args) {
|
||||||
|
try (InputStream is = FitElevationAnalysis.class.getResourceAsStream("/69287079d5e0a4532ba818ee.fit")) {
|
||||||
|
if (is == null) {
|
||||||
|
System.out.println("File not found on classpath");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Decode decode = new Decode();
|
||||||
|
MesgBroadcaster broadcaster = new MesgBroadcaster(decode);
|
||||||
|
|
||||||
|
// Collect session data
|
||||||
|
final Integer[] sessionAscent = {null};
|
||||||
|
final Integer[] sessionDescent = {null};
|
||||||
|
|
||||||
|
// Collect track point elevations
|
||||||
|
List<Double> elevations = new ArrayList<>();
|
||||||
|
|
||||||
|
broadcaster.addListener((SessionMesgListener) session -> {
|
||||||
|
sessionAscent[0] = session.getTotalAscent();
|
||||||
|
sessionDescent[0] = session.getTotalDescent();
|
||||||
|
System.out.println("=== SESSION DATA ===");
|
||||||
|
System.out.println("Total Ascent (raw): " + session.getTotalAscent() + " (type: " +
|
||||||
|
(session.getTotalAscent() != null ? session.getTotalAscent().getClass().getSimpleName() : "null") + ")");
|
||||||
|
System.out.println("Total Descent (raw): " + session.getTotalDescent() + " (type: " +
|
||||||
|
(session.getTotalDescent() != null ? session.getTotalDescent().getClass().getSimpleName() : "null") + ")");
|
||||||
|
System.out.println("Sport: " + session.getSport());
|
||||||
|
System.out.println("Sub Sport: " + session.getSubSport());
|
||||||
|
});
|
||||||
|
|
||||||
|
broadcaster.addListener((RecordMesgListener) record -> {
|
||||||
|
if (record.getAltitude() != null) {
|
||||||
|
elevations.add(record.getAltitude().doubleValue());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
decode.read(is, broadcaster);
|
||||||
|
|
||||||
|
System.out.println("\n=== TRACK POINT ELEVATION ANALYSIS ===");
|
||||||
|
System.out.println("Total track points with elevation: " + elevations.size());
|
||||||
|
|
||||||
|
if (elevations.isEmpty()) {
|
||||||
|
System.out.println("No elevation data in track points!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println("First elevation: " + elevations.get(0) + "m");
|
||||||
|
System.out.println("Last elevation: " + elevations.get(elevations.size() - 1) + "m");
|
||||||
|
|
||||||
|
// Find min/max
|
||||||
|
double min = elevations.stream().mapToDouble(d -> d).min().orElse(0);
|
||||||
|
double max = elevations.stream().mapToDouble(d -> d).max().orElse(0);
|
||||||
|
System.out.println("Min elevation: " + min + "m");
|
||||||
|
System.out.println("Max elevation: " + max + "m");
|
||||||
|
|
||||||
|
// Calculate gain/loss with different thresholds
|
||||||
|
for (double threshold : new double[]{0, 1.0, 2.0, 3.0, 5.0}) {
|
||||||
|
double gain = 0, loss = 0;
|
||||||
|
for (int i = 1; i < elevations.size(); i++) {
|
||||||
|
double delta = elevations.get(i) - elevations.get(i - 1);
|
||||||
|
if (Math.abs(delta) > threshold) {
|
||||||
|
if (delta > 0) gain += delta;
|
||||||
|
else loss += Math.abs(delta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
System.out.printf("Threshold %.1fm: gain=%.1fm, loss=%.1fm%n", threshold, gain, loss);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show first 20 elevation deltas to check for anomalies
|
||||||
|
System.out.println("\n=== FIRST 20 ELEVATION CHANGES ===");
|
||||||
|
for (int i = 1; i < Math.min(21, elevations.size()); i++) {
|
||||||
|
double delta = elevations.get(i) - elevations.get(i - 1);
|
||||||
|
System.out.printf(" Point %d: %.2fm -> %.2fm (delta: %+.2fm)%n",
|
||||||
|
i, elevations.get(i - 1), elevations.get(i), delta);
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println("\n=== COMPARISON ===");
|
||||||
|
System.out.println("Session ascent: " + sessionAscent[0] + "m");
|
||||||
|
System.out.println("Session descent: " + sessionDescent[0] + "m");
|
||||||
|
double gain2m = 0;
|
||||||
|
for (int i = 1; i < elevations.size(); i++) {
|
||||||
|
double delta = elevations.get(i) - elevations.get(i - 1);
|
||||||
|
if (delta > 2.0) gain2m += delta;
|
||||||
|
}
|
||||||
|
System.out.printf("Calculated gain (2m threshold): %.1fm%n", gain2m);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue