feat(komoot): improve import flow with throttling, cancellation, and UI guidance

Signed-off-by: Marcus Fihlon <marcus@fihlon.swiss>

src/main/java/net/javahippie/fitpub/service/KomootImportService.java

@@ -61,6 +61,15 @@ public class KomootImportService {
     @Value("${fitpub.komoot.base-url:https://www.komoot.com}")
     private String komootBaseUrl;
 
+    @Value("${fitpub.komoot.paginated-request-delay-ms:1000}")
+    private long paginatedRequestDelayMillis;
+
+    @Value("${fitpub.komoot.detail-to-gpx-delay-ms:500}")
+    private long detailToGpxDelayMillis;
+
+    @Value("${fitpub.komoot.activity-import-delay-ms:3000}")
+    private long activityImportDelayMillis;
+
     public KomootActivitiesResponse fetchCompletedActivities(KomootImportRequest request, UUID fitPubUserId) {
         List<KomootActivitySummaryDTO> activities = new ArrayList<>();
         Set<Long> importedKomootActivityIds = new HashSet<>(
@@ -80,6 +89,9 @@ public class KomootImportService {
                 }
                 extractActivities(root, activities, importedKomootActivityIds);
                 nextUri = extractNextUri(root);
+                if (nextUri != null) {
+                    pauseBeforeNextPageRequest();
+                }
             }
         } catch (HttpClientErrorException.Unauthorized | HttpClientErrorException.Forbidden e) {
             throw new IllegalArgumentException("Komoot login failed. Check email, password and Komoot ID.", e);
@@ -95,6 +107,10 @@ public class KomootImportService {
         return new KomootActivitiesResponse(request.userId(), activities.size(), activities);
     }
 
+    void pauseBeforeNextPageRequest() {
+        pause(paginatedRequestDelayMillis, "Interrupted while throttling paginated Komoot requests.");
+    }
+
     public KomootImportExecutionResponse importActivity(KomootActivityImportRequest request, UUID fitPubUserId) {
         if (activityRepository.findByUserIdAndKomootActivityId(fitPubUserId, request.activityId()).isPresent()) {
             return new KomootImportExecutionResponse(
@@ -106,6 +122,7 @@ public class KomootImportService {
         }
 
         JsonNode details = fetchActivityDetails(request.email(), request.password(), request.activityId());
+        pauseBetweenDetailAndGpxRequest();
         byte[] gpxData = fetchActivityGpx(request.email(), request.password(), request.activityId());
 
         ByteArrayMultipartFile gpxFile = new ByteArrayMultipartFile(
@@ -145,6 +162,8 @@ public class KomootImportService {
                 importedActivity.getActivityType()
         );
 
+        pauseAfterActivityImport();
+
         return new KomootImportExecutionResponse(
                 importedActivity.getId(),
                 request.activityId(),
@@ -153,6 +172,14 @@ public class KomootImportService {
         );
     }
 
+    void pauseBetweenDetailAndGpxRequest() {
+        pause(detailToGpxDelayMillis, "Interrupted while throttling Komoot detail and GPX requests.");
+    }
+
+    void pauseAfterActivityImport() {
+        pause(activityImportDelayMillis, "Interrupted while throttling Komoot activity imports.");
+    }
+
     private URI buildInitialUri(KomootImportRequest request) {
         String normalizedBaseUrl = komootBaseUrl.endsWith("/") ? komootBaseUrl.substring(0, komootBaseUrl.length() - 1) : komootBaseUrl;
         UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(normalizedBaseUrl + "/api/v007/users/" + request.userId() + "/tours/")
@@ -410,4 +437,17 @@ public class KomootImportService {
         JsonNode value = node.get(field);
         return value == null || value.isNull() ? null : value.asInt();
     }
+
+    private void pause(long delayMillis, String interruptedMessage) {
+        if (delayMillis <= 0) {
+            return;
+        }
+
+        try {
+            Thread.sleep(delayMillis);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException(interruptedMessage, e);
+        }
+    }
 }

src/main/resources/application.yml

@@ -106,6 +106,9 @@ fitpub:
 
   komoot:
     base-url: ${KOMOOT_BASE_URL:https://www.komoot.com}
+    paginated-request-delay-ms: ${KOMOOT_PAGINATED_REQUEST_DELAY_MS:1000}
+    detail-to-gpx-delay-ms: ${KOMOOT_DETAIL_TO_GPX_DELAY_MS:500}
+    activity-import-delay-ms: ${KOMOOT_ACTIVITY_IMPORT_DELAY_MS:3000}
 
 # Logging configuration
 logging:

src/main/resources/templates/activities/komoot.html

@@ -19,15 +19,20 @@
 
             <div class="alert alert-secondary">
                 <div class="fw-semibold mb-1">Important</div>
-                <div class="small mb-2">
+                <div class="small mb-1">
                     Your Komoot credentials are only used for this request and are not stored in FitPub.
                 </div>
                 <div class="small mb-0">
-                    Komoot does not provide a public API for this flow. This import currently depends on the
+                    The import currently runs in this browser tab. If you leave or reload the page, remaining activities will not continue importing automatically.
-                    same web API endpoints used by the Komoot website and may stop working if Komoot changes them.
                 </div>
             </div>
 
+            <ul class="small text-muted ps-3 mb-4">
+                <li>Import starts with the oldest new activities, so progress begins at the bottom of the list.</li>
+                <li>FitPub adds short delays between Komoot requests during loading and import to reduce rate limiting.</li>
+                <li>This integration depends on Komoot web endpoints and may stop working if Komoot changes them.</li>
+            </ul>
+
             <div id="errorAlert" class="alert alert-danger d-none" role="alert"></div>
 
             <div class="card shadow-sm">
@@ -81,6 +86,9 @@
                                     Importing...
                                 </span>
                             </button>
+                            <button type="button" class="btn btn-outline-danger d-none" id="cancelImportBtn">
+                                <i class="bi bi-stop-circle"></i> Stop After Current Activity
+                            </button>
                         </div>
                     </form>
                 </div>
@@ -130,10 +138,13 @@
             const importFirstBtn = document.getElementById('importFirstBtn');
             const importFirstText = document.getElementById('importFirstText');
             const importFirstSpinner = document.getElementById('importFirstSpinner');
+            const cancelImportBtn = document.getElementById('cancelImportBtn');
             let currentActivities = [];
+            let importCancellationRequested = false;
+            let importInProgress = false;
 
             function updateImportButtonState() {
-                importFirstBtn.disabled = currentActivities.length === 0;
+                importFirstBtn.disabled = importInProgress || currentActivities.length === 0;
             }
 
             function setLoading(loading) {
@@ -143,9 +154,13 @@
             }
 
             function setImportLoading(loading) {
-                importFirstBtn.disabled = loading;
+                importInProgress = loading;
+                loadActivitiesBtn.disabled = loading;
+                updateImportButtonState();
                 importFirstText.classList.toggle('d-none', loading);
                 importFirstSpinner.classList.toggle('d-none', !loading);
+                cancelImportBtn.classList.toggle('d-none', !loading);
+                cancelImportBtn.disabled = !loading;
             }
 
             function showError(message) {
@@ -222,6 +237,10 @@
             }
 
             function renderImportStatus(activity) {
+                if (activity.uiImportStatus === 'queued') {
+                    return '<i class="bi bi-hourglass-split text-warning" title="Queued for import" aria-label="Queued for import"></i>';
+                }
+
                 if (activity.uiImportStatus === 'importing') {
                     return '<span class="spinner-border spinner-border-sm text-primary" role="status" aria-label="Importing"></span>';
                 }
@@ -289,6 +308,15 @@
                 return Boolean(payload.startDate) !== Boolean(payload.endDate);
             }
 
+            function resetQueuedActivities() {
+                for (const activity of currentActivities) {
+                    if (activity.uiImportStatus === 'queued') {
+                        activity.uiImportStatus = null;
+                        activity.uiImportError = null;
+                    }
+                }
+            }
+
             form.addEventListener('submit', async function(event) {
                 event.preventDefault();
                 clearError();
@@ -356,6 +384,7 @@
                     showError('Load Komoot activities before starting the import.');
                     return;
                 }
+                importCancellationRequested = false;
                 setImportLoading(true);
 
                 try {
@@ -368,8 +397,15 @@
                         return;
                     }
 
+                    for (const activity of activitiesToImport) {
+                        activity.uiImportStatus = 'queued';
+                        activity.uiImportError = null;
+                    }
+                    renderActivities(currentActivities);
+
                     let importedCount = 0;
                     let failedCount = 0;
+                    let cancelled = false;
 
                     for (const activity of activitiesToImport) {
                         activity.uiImportStatus = 'importing';
@@ -407,9 +443,20 @@
                         }
 
                         renderActivities(currentActivities);
+
+                        if (importCancellationRequested) {
+                            cancelled = true;
+                            resetQueuedActivities();
+                            renderActivities(currentActivities);
+                            break;
+                        }
                     }
 
-                    showStatus(`Imported ${importedCount} Komoot activit${importedCount === 1 ? 'y' : 'ies'}${failedCount > 0 ? `, ${failedCount} failed` : ''}.`);
+                    if (cancelled) {
+                        showStatus(`Import stopped after the current activity. Imported ${importedCount} Komoot activit${importedCount === 1 ? 'y' : 'ies'}${failedCount > 0 ? `, ${failedCount} failed` : ''}.`);
+                    } else {
+                        showStatus(`Imported ${importedCount} Komoot activit${importedCount === 1 ? 'y' : 'ies'}${failedCount > 0 ? `, ${failedCount} failed` : ''}.`);
+                    }
                 } catch (error) {
                     let message = error instanceof Error ? error.message : 'Failed to import Komoot activities.';
 
@@ -420,9 +467,16 @@
                     showError(message);
                 } finally {
                     setImportLoading(false);
+                    importCancellationRequested = false;
                 }
             });
 
+            cancelImportBtn.addEventListener('click', function() {
+                importCancellationRequested = true;
+                cancelImportBtn.disabled = true;
+                showStatus('Import will stop after the current activity finishes.');
+            });
+
             updateImportButtonState();
         });
     </script>

src/test/java/net/javahippie/fitpub/service/KomootImportServiceTest.java

@@ -28,7 +28,9 @@ import java.util.UUID;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.springframework.test.web.client.ExpectedCount.once;
@@ -58,6 +60,9 @@ class KomootImportServiceTest {
         activityPostProcessingService = mock(ActivityPostProcessingService.class);
         service = new KomootImportService(restTemplate, activityRepository, activityFileService, activityPostProcessingService);
         ReflectionTestUtils.setField(service, "komootBaseUrl", "https://www.komoot.com");
+        ReflectionTestUtils.setField(service, "paginatedRequestDelayMillis", 0L);
+        ReflectionTestUtils.setField(service, "detailToGpxDelayMillis", 0L);
+        ReflectionTestUtils.setField(service, "activityImportDelayMillis", 0L);
     }
 
     @AfterEach
@@ -70,6 +75,8 @@ class KomootImportServiceTest {
         String authHeader = "Basic " + Base64.getEncoder()
                 .encodeToString("user@example.com:secret".getBytes(StandardCharsets.UTF_8));
         UUID userId = UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
+        KomootImportService throttledService = spy(service);
+        doNothing().when(throttledService).pauseBeforeNextPageRequest();
 
         when(activityRepository.findImportedKomootActivityIdsByUserId(userId)).thenReturn(List.of(1002L));
 
@@ -127,7 +134,7 @@ class KomootImportServiceTest {
                         }
                         """, MediaType.APPLICATION_JSON));
 
-        KomootActivitiesResponse response = service.fetchCompletedActivities(
+        KomootActivitiesResponse response = throttledService.fetchCompletedActivities(
                 new KomootImportRequest("user@example.com", "secret", "123456", null, null),
                 userId);
 
@@ -139,6 +146,7 @@ class KomootImportServiceTest {
         assertThat(response.activities().get(1).name()).isEqualTo("Lunch Walk");
         assertThat(response.activities().get(1).imported()).isTrue();
 
+        verify(throttledService).pauseBeforeNextPageRequest();
         server.verify();
     }
 
@@ -218,6 +226,9 @@ class KomootImportServiceTest {
                 .encodeToString("user@example.com:secret".getBytes(StandardCharsets.UTF_8));
         UUID userId = UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
         UUID importedActivityId = UUID.fromString("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb");
+        KomootImportService throttledService = spy(service);
+        doNothing().when(throttledService).pauseBetweenDetailAndGpxRequest();
+        doNothing().when(throttledService).pauseAfterActivityImport();
 
         when(activityRepository.findByUserIdAndKomootActivityId(userId, 2880957035L)).thenReturn(Optional.empty());
 
@@ -257,7 +268,7 @@ class KomootImportServiceTest {
         when(activityFileService.processActivityFile(any(), any(), any(), any(), any())).thenReturn(importedActivity);
         when(activityRepository.save(any(Activity.class))).thenAnswer(invocation -> invocation.getArgument(0));
 
-        KomootImportExecutionResponse response = service.importActivity(
+        KomootImportExecutionResponse response = throttledService.importActivity(
                 new KomootActivityImportRequest("user@example.com", "secret", "123456", 2880957035L),
                 userId
         );
@@ -271,6 +282,8 @@ class KomootImportServiceTest {
         assertThat(importedActivity.getVisibility()).isEqualTo(Activity.Visibility.PUBLIC);
         assertThat(importedActivity.getActivityType()).isEqualTo(Activity.ActivityType.RIDE);
 
+        verify(throttledService).pauseBetweenDetailAndGpxRequest();
+        verify(throttledService).pauseAfterActivityImport();
         verify(activityPostProcessingService).processActivityAsync(importedActivityId, userId);
         server.verify();
     }
@@ -302,6 +315,9 @@ class KomootImportServiceTest {
                 .encodeToString("user@example.com:secret".getBytes(StandardCharsets.UTF_8));
         UUID userId = UUID.fromString("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
         UUID importedActivityId = UUID.fromString("cccccccc-cccc-cccc-cccc-cccccccccccc");
+        KomootImportService throttledService = spy(service);
+        doNothing().when(throttledService).pauseBetweenDetailAndGpxRequest();
+        doNothing().when(throttledService).pauseAfterActivityImport();
 
         when(activityRepository.findByUserIdAndKomootActivityId(userId, 2880957036L)).thenReturn(Optional.empty());
 
@@ -341,7 +357,7 @@ class KomootImportServiceTest {
         when(activityFileService.processActivityFile(any(), any(), any(), any(), any())).thenReturn(importedActivity);
         when(activityRepository.save(any(Activity.class))).thenAnswer(invocation -> invocation.getArgument(0));
 
-        KomootImportExecutionResponse response = service.importActivity(
+        KomootImportExecutionResponse response = throttledService.importActivity(
                 new KomootActivityImportRequest("user@example.com", "secret", "123456", 2880957036L),
                 userId
         );
@@ -350,6 +366,8 @@ class KomootImportServiceTest {
         assertThat(response.status()).isEqualTo("IMPORTED");
         assertThat(importedActivity.getActivityType()).isEqualTo(Activity.ActivityType.OTHER);
 
+        verify(throttledService).pauseBetweenDetailAndGpxRequest();
+        verify(throttledService).pauseAfterActivityImport();
         verify(activityPostProcessingService).processActivityAsync(importedActivityId, userId);
         server.verify();
     }