feat(komoot): improve import flow with throttling, cancellation, and UI guidance
Signed-off-by: Marcus Fihlon <marcus@fihlon.swiss>
This commit is contained in:
parent
0387ca01e3
commit
f7f919f0b1
GPG key ID:
C6B7F469EE363E1F
4 changed files with 124 additions and 9 deletions
|
|
@ -61,6 +61,15 @@ public class KomootImportService {
|
|||
@Value("${fitpub.komoot.base-url:https://www.komoot.com}")
|
||||
private String komootBaseUrl;
|
||||
|
||||
@Value("${fitpub.komoot.paginated-request-delay-ms:1000}")
|
||||
private long paginatedRequestDelayMillis;
|
||||
|
||||
@Value("${fitpub.komoot.detail-to-gpx-delay-ms:500}")
|
||||
private long detailToGpxDelayMillis;
|
||||
|
||||
@Value("${fitpub.komoot.activity-import-delay-ms:3000}")
|
||||
private long activityImportDelayMillis;
|
||||
|
||||
public KomootActivitiesResponse fetchCompletedActivities(KomootImportRequest request, UUID fitPubUserId) {
|
||||
List<KomootActivitySummaryDTO> activities = new ArrayList<>();
|
||||
Set<Long> importedKomootActivityIds = new HashSet<>(
|
||||
|
|
@ -80,6 +89,9 @@ public class KomootImportService {
|
|||
}
|
||||
extractActivities(root, activities, importedKomootActivityIds);
|
||||
nextUri = extractNextUri(root);
|
||||
if (nextUri != null) {
|
||||
pauseBeforeNextPageRequest();
|
||||
}
|
||||
}
|
||||
} catch (HttpClientErrorException.Unauthorized | HttpClientErrorException.Forbidden e) {
|
||||
throw new IllegalArgumentException("Komoot login failed. Check email, password and Komoot ID.", e);
|
||||
|
|
@ -95,6 +107,10 @@ public class KomootImportService {
|
|||
return new KomootActivitiesResponse(request.userId(), activities.size(), activities);
|
||||
}
|
||||
|
||||
void pauseBeforeNextPageRequest() {
|
||||
pause(paginatedRequestDelayMillis, "Interrupted while throttling paginated Komoot requests.");
|
||||
}
|
||||
|
||||
public KomootImportExecutionResponse importActivity(KomootActivityImportRequest request, UUID fitPubUserId) {
|
||||
if (activityRepository.findByUserIdAndKomootActivityId(fitPubUserId, request.activityId()).isPresent()) {
|
||||
return new KomootImportExecutionResponse(
|
||||
|
|
@ -106,6 +122,7 @@ public class KomootImportService {
|
|||
}
|
||||
|
||||
JsonNode details = fetchActivityDetails(request.email(), request.password(), request.activityId());
|
||||
pauseBetweenDetailAndGpxRequest();
|
||||
byte[] gpxData = fetchActivityGpx(request.email(), request.password(), request.activityId());
|
||||
|
||||
ByteArrayMultipartFile gpxFile = new ByteArrayMultipartFile(
|
||||
|
|
@ -145,6 +162,8 @@ public class KomootImportService {
|
|||
importedActivity.getActivityType()
|
||||
);
|
||||
|
||||
pauseAfterActivityImport();
|
||||
|
||||
return new KomootImportExecutionResponse(
|
||||
importedActivity.getId(),
|
||||
request.activityId(),
|
||||
|
|
@ -153,6 +172,14 @@ public class KomootImportService {
|
|||
);
|
||||
}
|
||||
|
||||
void pauseBetweenDetailAndGpxRequest() {
|
||||
pause(detailToGpxDelayMillis, "Interrupted while throttling Komoot detail and GPX requests.");
|
||||
}
|
||||
|
||||
void pauseAfterActivityImport() {
|
||||
pause(activityImportDelayMillis, "Interrupted while throttling Komoot activity imports.");
|
||||
}
|
||||
|
||||
private URI buildInitialUri(KomootImportRequest request) {
|
||||
String normalizedBaseUrl = komootBaseUrl.endsWith("/") ? komootBaseUrl.substring(0, komootBaseUrl.length() - 1) : komootBaseUrl;
|
||||
UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(normalizedBaseUrl + "/api/v007/users/" + request.userId() + "/tours/")
|
||||
|
|
@ -410,4 +437,17 @@ public class KomootImportService {
|
|||
JsonNode value = node.get(field);
|
||||
return value == null || value.isNull() ? null : value.asInt();
|
||||
}
|
||||
|
||||
private void pause(long delayMillis, String interruptedMessage) {
|
||||
if (delayMillis <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
Thread.sleep(delayMillis);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException(interruptedMessage, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,6 +106,9 @@ fitpub:
|
|||
|
||||
komoot:
|
||||
base-url: ${KOMOOT_BASE_URL:https://www.komoot.com}
|
||||
paginated-request-delay-ms: ${KOMOOT_PAGINATED_REQUEST_DELAY_MS:1000}
|
||||
detail-to-gpx-delay-ms: ${KOMOOT_DETAIL_TO_GPX_DELAY_MS:500}
|
||||
activity-import-delay-ms: ${KOMOOT_ACTIVITY_IMPORT_DELAY_MS:3000}
|
||||
|
||||
# Logging configuration
|
||||
logging:
|
||||
|
|
|
|||
|
|
@ -19,15 +19,20 @@
|
|||
|
||||
<div class="alert alert-secondary">
|
||||
<div class="fw-semibold mb-1">Important</div>
|
||||
<div class="small mb-2">
|
||||
<div class="small mb-1">
|
||||
Your Komoot credentials are only used for this request and are not stored in FitPub.
|
||||
</div>
|
||||
<div class="small mb-0">
|
||||
Komoot does not provide a public API for this flow. This import currently depends on the
|
||||
same web API endpoints used by the Komoot website and may stop working if Komoot changes them.
|
||||
The import currently runs in this browser tab. If you leave or reload the page, remaining activities will not continue importing automatically.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ul class="small text-muted ps-3 mb-4">
|
||||
<li>Import starts with the oldest new activities, so progress begins at the bottom of the list.</li>
|
||||
<li>FitPub adds short delays between Komoot requests during loading and import to reduce rate limiting.</li>
|
||||
<li>This integration depends on Komoot web endpoints and may stop working if Komoot changes them.</li>
|
||||
</ul>
|
||||
|
||||
<div id="errorAlert" class="alert alert-danger d-none" role="alert"></div>
|
||||
|
||||
<div class="card shadow-sm">
|
||||
|
|
@ -81,6 +86,9 @@
|
|||
Importing...
|
||||
</span>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger d-none" id="cancelImportBtn">
|
||||
<i class="bi bi-stop-circle"></i> Stop After Current Activity
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
|
@ -130,10 +138,13 @@
|
|||
const importFirstBtn = document.getElementById('importFirstBtn');
|
||||
const importFirstText = document.getElementById('importFirstText');
|
||||
const importFirstSpinner = document.getElementById('importFirstSpinner');
|
||||
const cancelImportBtn = document.getElementById('cancelImportBtn');
|
||||
let currentActivities = [];
|
||||
let importCancellationRequested = false;
|
||||
let importInProgress = false;
|
||||
|
||||
function updateImportButtonState() {
|
||||
importFirstBtn.disabled = currentActivities.length === 0;
|
||||
importFirstBtn.disabled = importInProgress || currentActivities.length === 0;
|
||||
}
|
||||
|
||||
function setLoading(loading) {
|
||||
|
|
@ -143,9 +154,13 @@
|
|||
}
|
||||
|
||||
function setImportLoading(loading) {
|
||||
importFirstBtn.disabled = loading;
|
||||
importInProgress = loading;
|
||||
loadActivitiesBtn.disabled = loading;
|
||||
updateImportButtonState();
|
||||
importFirstText.classList.toggle('d-none', loading);
|
||||
importFirstSpinner.classList.toggle('d-none', !loading);
|
||||
cancelImportBtn.classList.toggle('d-none', !loading);
|
||||
cancelImportBtn.disabled = !loading;
|
||||
}
|
||||
|
||||
function showError(message) {
|
||||
|
|
@ -222,6 +237,10 @@
|
|||
}
|
||||
|
||||
function renderImportStatus(activity) {
|
||||
if (activity.uiImportStatus === 'queued') {
|
||||
return '<i class="bi bi-hourglass-split text-warning" title="Queued for import" aria-label="Queued for import"></i>';
|
||||
}
|
||||
|
||||
if (activity.uiImportStatus === 'importing') {
|
||||
return '<span class="spinner-border spinner-border-sm text-primary" role="status" aria-label="Importing"></span>';
|
||||
}
|
||||
|
|
@ -289,6 +308,15 @@
|
|||
return Boolean(payload.startDate) !== Boolean(payload.endDate);
|
||||
}
|
||||
|
||||
function resetQueuedActivities() {
|
||||
for (const activity of currentActivities) {
|
||||
if (activity.uiImportStatus === 'queued') {
|
||||
activity.uiImportStatus = null;
|
||||
activity.uiImportError = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
form.addEventListener('submit', async function(event) {
|
||||
event.preventDefault();
|
||||
clearError();
|
||||
|
|
@ -356,6 +384,7 @@
|
|||
showError('Load Komoot activities before starting the import.');
|
||||
return;
|
||||
}
|
||||
importCancellationRequested = false;
|
||||
setImportLoading(true);
|
||||
|
||||
try {
|
||||
|
|
@ -368,8 +397,15 @@
|
|||
return;
|
||||
}
|
||||
|
||||
for (const activity of activitiesToImport) {
|
||||
activity.uiImportStatus = 'queued';
|
||||
activity.uiImportError = null;
|
||||
}
|
||||
renderActivities(currentActivities);
|
||||
|
||||
let importedCount = 0;
|
||||
let failedCount = 0;
|
||||
let cancelled = false;
|
||||
|
||||
for (const activity of activitiesToImport) {
|
||||
activity.uiImportStatus = 'importing';
|
||||
|
|
@ -407,9 +443,20 @@
|
|||
}
|
||||
|
||||
renderActivities(currentActivities);
|
||||
|
||||
if (importCancellationRequested) {
|
||||
cancelled = true;
|
||||
resetQueuedActivities();
|
||||
renderActivities(currentActivities);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
showStatus(`Imported ${importedCount} Komoot activit${importedCount === 1 ? 'y' : 'ies'}${failedCount > 0 ? `, ${failedCount} failed` : ''}.`);
|
||||
if (cancelled) {
|
||||
showStatus(`Import stopped after the current activity. Imported ${importedCount} Komoot activit${importedCount === 1 ? 'y' : 'ies'}${failedCount > 0 ? `, ${failedCount} failed` : ''}.`);
|
||||
} else {
|
||||
showStatus(`Imported ${importedCount} Komoot activit${importedCount === 1 ? 'y' : 'ies'}${failedCount > 0 ? `, ${failedCount} failed` : ''}.`);
|
||||
}
|
||||
} catch (error) {
|
||||
let message = error instanceof Error ? error.message : 'Failed to import Komoot activities.';
|
||||
|
||||
|
|
@ -420,9 +467,16 @@
|
|||
showError(message);
|
||||
} finally {
|
||||
setImportLoading(false);
|
||||
importCancellationRequested = false;
|
||||
}
|
||||
});
|
||||
|
||||
cancelImportBtn.addEventListener('click', function() {
|
||||
importCancellationRequested = true;
|
||||
cancelImportBtn.disabled = true;
|
||||
showStatus('Import will stop after the current activity finishes.');
|
||||
});
|
||||
|
||||
updateImportButtonState();
|
||||
});
|
||||
</script>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ import java.util.UUID;
|
|||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doNothing;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.client.ExpectedCount.once;
|
||||
|
|
@ -58,6 +60,9 @@ class KomootImportServiceTest {
|
|||
activityPostProcessingService = mock(ActivityPostProcessingService.class);
|
||||
service = new KomootImportService(restTemplate, activityRepository, activityFileService, activityPostProcessingService);
|
||||
ReflectionTestUtils.setField(service, "komootBaseUrl", "https://www.komoot.com");
|
||||
ReflectionTestUtils.setField(service, "paginatedRequestDelayMillis", 0L);
|
||||
ReflectionTestUtils.setField(service, "detailToGpxDelayMillis", 0L);
|
||||
ReflectionTestUtils.setField(service, "activityImportDelayMillis", 0L);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
|
|
@ -70,6 +75,8 @@ class KomootImportServiceTest {
|
|||
String authHeader = "Basic " + Base64.getEncoder()
|
||||
.encodeToString("user@example.com:secret".getBytes(StandardCharsets.UTF_8));
|
||||
UUID userId = UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
|
||||
KomootImportService throttledService = spy(service);
|
||||
doNothing().when(throttledService).pauseBeforeNextPageRequest();
|
||||
|
||||
when(activityRepository.findImportedKomootActivityIdsByUserId(userId)).thenReturn(List.of(1002L));
|
||||
|
||||
|
|
@ -127,7 +134,7 @@ class KomootImportServiceTest {
|
|||
}
|
||||
""", MediaType.APPLICATION_JSON));
|
||||
|
||||
KomootActivitiesResponse response = service.fetchCompletedActivities(
|
||||
KomootActivitiesResponse response = throttledService.fetchCompletedActivities(
|
||||
new KomootImportRequest("user@example.com", "secret", "123456", null, null),
|
||||
userId);
|
||||
|
||||
|
|
@ -139,6 +146,7 @@ class KomootImportServiceTest {
|
|||
assertThat(response.activities().get(1).name()).isEqualTo("Lunch Walk");
|
||||
assertThat(response.activities().get(1).imported()).isTrue();
|
||||
|
||||
verify(throttledService).pauseBeforeNextPageRequest();
|
||||
server.verify();
|
||||
}
|
||||
|
||||
|
|
@ -218,6 +226,9 @@ class KomootImportServiceTest {
|
|||
.encodeToString("user@example.com:secret".getBytes(StandardCharsets.UTF_8));
|
||||
UUID userId = UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
|
||||
UUID importedActivityId = UUID.fromString("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb");
|
||||
KomootImportService throttledService = spy(service);
|
||||
doNothing().when(throttledService).pauseBetweenDetailAndGpxRequest();
|
||||
doNothing().when(throttledService).pauseAfterActivityImport();
|
||||
|
||||
when(activityRepository.findByUserIdAndKomootActivityId(userId, 2880957035L)).thenReturn(Optional.empty());
|
||||
|
||||
|
|
@ -257,7 +268,7 @@ class KomootImportServiceTest {
|
|||
when(activityFileService.processActivityFile(any(), any(), any(), any(), any())).thenReturn(importedActivity);
|
||||
when(activityRepository.save(any(Activity.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
KomootImportExecutionResponse response = service.importActivity(
|
||||
KomootImportExecutionResponse response = throttledService.importActivity(
|
||||
new KomootActivityImportRequest("user@example.com", "secret", "123456", 2880957035L),
|
||||
userId
|
||||
);
|
||||
|
|
@ -271,6 +282,8 @@ class KomootImportServiceTest {
|
|||
assertThat(importedActivity.getVisibility()).isEqualTo(Activity.Visibility.PUBLIC);
|
||||
assertThat(importedActivity.getActivityType()).isEqualTo(Activity.ActivityType.RIDE);
|
||||
|
||||
verify(throttledService).pauseBetweenDetailAndGpxRequest();
|
||||
verify(throttledService).pauseAfterActivityImport();
|
||||
verify(activityPostProcessingService).processActivityAsync(importedActivityId, userId);
|
||||
server.verify();
|
||||
}
|
||||
|
|
@ -302,6 +315,9 @@ class KomootImportServiceTest {
|
|||
.encodeToString("user@example.com:secret".getBytes(StandardCharsets.UTF_8));
|
||||
UUID userId = UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
|
||||
UUID importedActivityId = UUID.fromString("cccccccc-cccc-cccc-cccc-cccccccccccc");
|
||||
KomootImportService throttledService = spy(service);
|
||||
doNothing().when(throttledService).pauseBetweenDetailAndGpxRequest();
|
||||
doNothing().when(throttledService).pauseAfterActivityImport();
|
||||
|
||||
when(activityRepository.findByUserIdAndKomootActivityId(userId, 2880957036L)).thenReturn(Optional.empty());
|
||||
|
||||
|
|
@ -341,7 +357,7 @@ class KomootImportServiceTest {
|
|||
when(activityFileService.processActivityFile(any(), any(), any(), any(), any())).thenReturn(importedActivity);
|
||||
when(activityRepository.save(any(Activity.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
KomootImportExecutionResponse response = service.importActivity(
|
||||
KomootImportExecutionResponse response = throttledService.importActivity(
|
||||
new KomootActivityImportRequest("user@example.com", "secret", "123456", 2880957036L),
|
||||
userId
|
||||
);
|
||||
|
|
@ -350,6 +366,8 @@ class KomootImportServiceTest {
|
|||
assertThat(response.status()).isEqualTo("IMPORTED");
|
||||
assertThat(importedActivity.getActivityType()).isEqualTo(Activity.ActivityType.OTHER);
|
||||
|
||||
verify(throttledService).pauseBetweenDetailAndGpxRequest();
|
||||
verify(throttledService).pauseAfterActivityImport();
|
||||
verify(activityPostProcessingService).processActivityAsync(importedActivityId, userId);
|
||||
server.verify();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue