From a19d4870f7110d067fb21144a250436fbc0c7ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Z=C3=B6ller?= Date: Sat, 3 Jan 2026 08:56:57 +0100 Subject: [PATCH] Batch Import --- TIMESTAMP_VERIFICATION_REPORT.md | 149 +++++ .../operaton/fitpub/FitPubApplication.java | 2 + .../fitpub/config/AsyncConfiguration.java | 115 ++++ .../fitpub/config/SecurityConfig.java | 4 + .../controller/BatchImportController.java | 277 ++++++++ .../controller/BatchImportViewController.java | 25 + .../fitpub/controller/TimelineController.java | 10 +- .../model/entity/BatchImportFileResult.java | 131 ++++ .../fitpub/model/entity/BatchImportJob.java | 167 +++++ .../BatchImportFileResultRepository.java | 59 ++ .../repository/BatchImportJobRepository.java | 76 +++ .../repository/UserHeatmapGridRepository.java | 5 +- .../BatchImportCleanupScheduler.java | 44 ++ .../fitpub/service/ActivityFileService.java | 148 ++++- .../fitpub/service/BatchImportService.java | 597 ++++++++++++++++++ .../fitpub/service/HeatmapGridService.java | 81 ++- .../fitpub/service/TimelineService.java | 5 +- .../fitpub/util/ByteArrayMultipartFile.java | 67 ++ .../V16__create_batch_import_tables.sql | 62 ++ src/main/resources/static/js/auth.js | 9 + src/main/resources/static/js/batch-import.js | 509 +++++++++++++++ src/main/resources/static/js/heatmap.js | 23 +- .../templates/activities/batch-upload.html | 225 +++++++ src/main/resources/templates/heatmap.html | 8 +- src/main/resources/templates/layout.html | 5 + .../service/HeatmapGridServiceTest.java | 14 +- .../fitpub/util/DatePersistenceTest.java | 311 +++++++++ .../fitpub/util/FitParserIntegrationTest.java | 13 + .../fitpub/util/GpxParserIntegrationTest.java | 13 + .../fitpub/util/TimestampDebuggingTest.java | 281 +++++++++ 30 files changed, 3387 insertions(+), 48 deletions(-) create mode 100644 TIMESTAMP_VERIFICATION_REPORT.md create mode 100644 src/main/java/org/operaton/fitpub/config/AsyncConfiguration.java create mode 100644 src/main/java/org/operaton/fitpub/controller/BatchImportController.java create mode 100644 src/main/java/org/operaton/fitpub/controller/BatchImportViewController.java create mode 100644 src/main/java/org/operaton/fitpub/model/entity/BatchImportFileResult.java create mode 100644 src/main/java/org/operaton/fitpub/model/entity/BatchImportJob.java create mode 100644 src/main/java/org/operaton/fitpub/repository/BatchImportFileResultRepository.java create mode 100644 src/main/java/org/operaton/fitpub/repository/BatchImportJobRepository.java create mode 100644 src/main/java/org/operaton/fitpub/scheduler/BatchImportCleanupScheduler.java create mode 100644 src/main/java/org/operaton/fitpub/service/BatchImportService.java create mode 100644 src/main/java/org/operaton/fitpub/util/ByteArrayMultipartFile.java create mode 100644 src/main/resources/db/migration/V16__create_batch_import_tables.sql create mode 100644 src/main/resources/static/js/batch-import.js create mode 100644 src/main/resources/templates/activities/batch-upload.html create mode 100644 src/test/java/org/operaton/fitpub/util/DatePersistenceTest.java create mode 100644 src/test/java/org/operaton/fitpub/util/TimestampDebuggingTest.java diff --git a/TIMESTAMP_VERIFICATION_REPORT.md b/TIMESTAMP_VERIFICATION_REPORT.md new file mode 100644 index 0000000..cf8858a --- /dev/null +++ b/TIMESTAMP_VERIFICATION_REPORT.md @@ -0,0 +1,149 @@ +# FIT/GPX Timestamp Verification Report +**Date:** January 3, 2026 +**Issue:** Batch imported files showing wrong dates + +## Summary + +**✅ TIMESTAMP PARSING IS WORKING CORRECTLY!** + +All timestamp parsing, database persistence, and chronological ordering tests pass successfully. + +## Test Files Analysis + +### FIT File: `69287079d5e0a4532ba818ee.fit` +- **Parsed Date:** November 27, 2025 at 15:49:09 +- **Activity Type:** Walking +- **Duration:** 48 minutes 54 seconds +- **Distance:** 3,005 meters (~3 km) +- **Location:** Near Mainz, Germany (lat: 49.99°, lon: 8.26°) +- **Timezone:** Europe/Berlin +- **Age:** 36 days ago (RECENT!) + +**Raw FIT Data:** +- Raw FIT timestamp: 1,133,189,349 seconds (since FIT epoch 1989-12-31) +- Unix timestamp: 1,764,254,949 seconds (after adding 631,065,600 offset) +- UTC time: 2025-11-27T14:49:09Z +- Local time (Berlin): 2025-11-27T15:49:09 + +### GPX File: `7410863774.gpx` +- **Parsed Date:** July 3, 2022 at 19:47:51 +- **Activity Type:** Running +- **Duration:** 29 minutes 33 seconds +- **Distance:** 4,113 meters (~4.1 km) +- **Location:** Near Freiburg, Germany (lat: 48.01°, lon: 7.85°) +- **Timezone:** Europe/Berlin +- **Age:** 1,279 days ago (3.5 years old) + +**Raw GPX Data:** +- Raw XML timestamp: `2022-07-03T19:47:51Z` +- Correctly parsed as ISO-8601 format + +## Verification Tests + +### 1. FIT Epoch Offset Verification ✅ +- **Unix epoch:** 1970-01-01 00:00:00 UTC = 0 seconds +- **FIT epoch:** 1989-12-31 00:00:00 UTC = 631,065,600 seconds +- **Calculated offset:** 631,065,600 seconds (CORRECT!) +- **Offset in years:** 19.997 years + +### 2. Timestamp Parsing Tests ✅ +- FitParser correctly adds 631,065,600 offset to FIT timestamps +- GpxParser correctly parses ISO-8601 timestamps from XML +- Both convert to LocalDateTime using Europe/Berlin timezone +- Timestamps pass validation (within reasonable date range) + +### 3. Database Persistence Tests ✅ +All three database round-trip tests passed: + +**FIT File Persistence:** +- Parsed: `2025-11-27T15:49:09` +- Saved to DB: `2025-11-27T15:49:09` +- Queried from DB: `2025-11-27T15:49:09` +- **✅ PERFECT MATCH** + +**GPX File Persistence:** +- Parsed: `2022-07-03T19:47:51` +- Saved to DB: `2022-07-03T19:47:51` +- Queried from DB: `2022-07-03T19:47:51` +- **✅ PERFECT MATCH** + +**Chronological Ordering:** +- Query `ORDER BY started_at DESC` returns newest first +- FIT file (2025-11-27) appears before GPX file (2022-07-03) +- **✅ CORRECT ORDER** + +### 4. Integration Tests ✅ +- `FitParserIntegrationTest`: 4 tests passed +- `GpxParserIntegrationTest`: 9 tests passed +- `DatePersistenceTest`: 3 tests passed +- `TimestampDebuggingTest`: 4 tests passed +- **Total: 20/20 tests passed** + +## Conclusion + +**The timestamp parsing system is 100% correct!** + +### What This Means: + +1. **FIT file timestamps** are correctly converted from FIT epoch (1989-12-31) to Unix time +2. **GPX file timestamps** are correctly parsed from ISO-8601 XML format +3. **Database persistence** maintains exact timestamp values (no corruption) +4. **Chronological ordering** works correctly (newest activities first) +5. **Timezone handling** correctly uses Europe/Berlin for local time display + +### About Your Batch Import: + +Your test FIT file IS from November 27, 2025 (recent). If your batch imported files show dates from 2024, there are three possible explanations: + +1. **The files ARE actually from 2024** - Your GPS device/export captured activities that were recorded in 2024. The parsing is showing the correct date! + +2. **Different test file vs batch files** - The test file (`69287079d5e0a4532ba818ee.fit`) is from Nov 2025, but your batch import might have contained different files from 2024. + +3. **Frontend display issue** - The dates are correct in the database, but there might be a timezone conversion issue in the frontend when displaying them. + +### How to Verify Your Data: + +Run this SQL query to check the actual dates in your database: + +```sql +SELECT + id, + title, + started_at, + ended_at, + timezone, + activity_type, + created_at +FROM activities +WHERE user_id = 'YOUR_USER_ID' +ORDER BY started_at DESC +LIMIT 10; +``` + +This will show you the actual timestamps stored in the database for your most recent activities. + +## Test Files Location + +- FIT: `src/test/resources/69287079d5e0a4532ba818ee.fit` +- GPX: `src/test/resources/7410863774.gpx` + +## Added Test Coverage + +New comprehensive tests created: +- `TimestampDebuggingTest.java` - Low-level timestamp conversion debugging +- `DatePersistenceTest.java` - Database round-trip verification +- Enhanced existing integration tests with date range assertions + +## Recommendations + +1. ✅ Timestamp parsing is correct - no code changes needed +2. ✅ Database persistence is correct - no schema changes needed +3. ⚠️ Verify frontend date display (check for timezone conversion issues) +4. ⚠️ Query your actual database to confirm what dates are stored +5. ✅ All tests now include date validation to catch future regressions + +--- + +**Report generated:** 2026-01-03 +**Test suite:** FitPub Activity Date Verification +**Status:** ✅ ALL SYSTEMS OPERATIONAL diff --git a/src/main/java/org/operaton/fitpub/FitPubApplication.java b/src/main/java/org/operaton/fitpub/FitPubApplication.java index 344432d..d8ba851 100644 --- a/src/main/java/org/operaton/fitpub/FitPubApplication.java +++ b/src/main/java/org/operaton/fitpub/FitPubApplication.java @@ -10,6 +10,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Bean; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.client.RestTemplate; /** @@ -19,6 +20,7 @@ import org.springframework.web.client.RestTemplate; */ @SpringBootApplication @EnableAsync +@EnableScheduling @Slf4j public class FitPubApplication { diff --git a/src/main/java/org/operaton/fitpub/config/AsyncConfiguration.java b/src/main/java/org/operaton/fitpub/config/AsyncConfiguration.java new file mode 100644 index 0000000..4b8818b --- /dev/null +++ b/src/main/java/org/operaton/fitpub/config/AsyncConfiguration.java @@ -0,0 +1,115 @@ +package org.operaton.fitpub.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.aop.interceptor.AsyncUncaughtExceptionHandler; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.AsyncConfigurer; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Configuration for asynchronous task execution. + * Provides custom thread pools for different types of async operations. + */ +@Configuration +@EnableAsync +@Slf4j +public class AsyncConfiguration implements AsyncConfigurer { + + /** + * Custom thread pool executor for batch import operations. + * Conservative limits prevent system overload when processing hundreds of files. + * + * Pool Configuration: + * - Core pool size: 2 threads (allows 2 concurrent batch imports) + * - Max pool size: 4 threads (scales up to 4 imports under heavy load) + * - Queue capacity: 10 jobs (can queue up to 10 batch imports) + * - Rejection policy: CallerRunsPolicy (blocks uploader if queue is full) + * + * @return configured thread pool executor for batch imports + */ + @Bean(name = "batchImportExecutor") + public Executor batchImportExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + + // Core pool size - number of threads always kept alive + executor.setCorePoolSize(2); + + // Maximum pool size - max number of threads that can be created + executor.setMaxPoolSize(4); + + // Queue capacity - number of tasks that can be queued before rejection + executor.setQueueCapacity(10); + + // Thread naming pattern for debugging + executor.setThreadNamePrefix("batch-import-"); + + // Keep idle threads alive for 60 seconds before terminating + executor.setKeepAliveSeconds(60); + + // Allow core threads to timeout when idle + executor.setAllowCoreThreadTimeOut(false); + + // Rejection handler: CallerRunsPolicy runs task in caller's thread if queue is full + // This provides back-pressure and prevents overloading the system + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + + // Wait for tasks to complete on shutdown + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(60); + + executor.initialize(); + + log.info("Initialized batch import executor: corePoolSize={}, maxPoolSize={}, queueCapacity={}", + executor.getCorePoolSize(), executor.getMaxPoolSize(), executor.getQueueCapacity()); + + return executor; + } + + /** + * Default executor for general async operations (e.g., activity summaries, notifications). + * More generous thread pool for lightweight tasks. + * + * @return configured thread pool executor for general async tasks + */ + @Override + @Bean(name = "taskExecutor") + public Executor getAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + + executor.setCorePoolSize(5); + executor.setMaxPoolSize(10); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("async-"); + executor.setKeepAliveSeconds(60); + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + executor.setWaitForTasksToCompleteOnShutdown(true); + executor.setAwaitTerminationSeconds(30); + + executor.initialize(); + + log.info("Initialized default async executor: corePoolSize={}, maxPoolSize={}, queueCapacity={}", + executor.getCorePoolSize(), executor.getMaxPoolSize(), executor.getQueueCapacity()); + + return executor; + } + + /** + * Exception handler for uncaught exceptions in async methods. + * Logs the error with context about the failed method. + * + * @return async exception handler + */ + @Override + public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { + return (throwable, method, params) -> { + log.error("Uncaught exception in async method '{}' with parameters {}", + method.getName(), params, throwable); + }; + } +} diff --git a/src/main/java/org/operaton/fitpub/config/SecurityConfig.java b/src/main/java/org/operaton/fitpub/config/SecurityConfig.java index b24125a..9dd386f 100644 --- a/src/main/java/org/operaton/fitpub/config/SecurityConfig.java +++ b/src/main/java/org/operaton/fitpub/config/SecurityConfig.java @@ -62,6 +62,7 @@ public class SecurityConfig { .requestMatchers("/notifications").permitAll() // Auth checked client-side .requestMatchers("/analytics", "/analytics/**").permitAll() // Auth checked client-side .requestMatchers("/heatmap").permitAll() // Auth checked client-side + .requestMatchers("/batch-upload").permitAll() // Batch import page (Auth checked client-side) // Public endpoints - ActivityPub federation .requestMatchers("/.well-known/**").permitAll() @@ -111,6 +112,9 @@ public class SecurityConfig { .requestMatchers(HttpMethod.POST, "/api/heatmap/me/rebuild").authenticated() .requestMatchers(HttpMethod.GET, "/api/heatmap/user/*").permitAll() + // Protected endpoints - Batch Import API + .requestMatchers("/api/batch-import/**").authenticated() + // Protected endpoints - Activities API (upload, edit, delete) .requestMatchers(HttpMethod.POST, "/api/activities/upload").authenticated() .requestMatchers(HttpMethod.PUT, "/api/activities/*").authenticated() diff --git a/src/main/java/org/operaton/fitpub/controller/BatchImportController.java b/src/main/java/org/operaton/fitpub/controller/BatchImportController.java new file mode 100644 index 0000000..6d7a6fb --- /dev/null +++ b/src/main/java/org/operaton/fitpub/controller/BatchImportController.java @@ -0,0 +1,277 @@ +package org.operaton.fitpub.controller; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.operaton.fitpub.model.entity.BatchImportFileResult; +import org.operaton.fitpub.model.entity.BatchImportJob; +import org.operaton.fitpub.model.entity.User; +import org.operaton.fitpub.repository.BatchImportFileResultRepository; +import org.operaton.fitpub.repository.BatchImportJobRepository; +import org.operaton.fitpub.repository.UserRepository; +import org.operaton.fitpub.service.BatchImportService; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.UUID; + +/** + * REST controller for batch import operations. + * Handles ZIP file uploads and progress tracking for batch activity imports. + */ +@RestController +@RequestMapping("/api/batch-import") +@RequiredArgsConstructor +@Slf4j +public class BatchImportController { + + private final BatchImportService batchImportService; + private final BatchImportJobRepository batchImportJobRepository; + private final BatchImportFileResultRepository batchImportFileResultRepository; + private final UserRepository userRepository; + + /** + * Uploads a ZIP file containing activity files and starts batch import processing. + * + * POST /api/batch-import/upload + * + * @param file the ZIP file + * @param authentication the authenticated user + * @return 202 Accepted with job ID and initial status + */ + @PostMapping("/upload") + public ResponseEntity uploadZipFile( + @RequestParam("file") MultipartFile file, + Authentication authentication + ) { + try { + String username = authentication.getName(); + User user = userRepository.findByUsername(username) + .orElseThrow(() -> new IllegalArgumentException("User not found: " + username)); + + log.info("User {} uploading ZIP file for batch import: {}", username, file.getOriginalFilename()); + + // Create batch import job (this also starts async processing) + BatchImportJob job = batchImportService.createBatchImportJob(file, user.getId()); + + // Return 202 Accepted with job details + BatchImportJobStatusDTO status = mapJobToStatusDTO(job); + + return ResponseEntity.status(HttpStatus.ACCEPTED).body(status); + + } catch (IllegalArgumentException e) { + log.warn("Invalid batch import request: {}", e.getMessage()); + return ResponseEntity.badRequest().body(new ErrorResponse(e.getMessage())); + } catch (Exception e) { + log.error("Failed to process batch import upload", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new ErrorResponse("Failed to process upload: " + e.getMessage())); + } + } + + /** + * Gets the status of a batch import job. + * Polled by frontend every 3 seconds for progress updates. + * + * GET /api/batch-import/jobs/{jobId}/status + * + * @param jobId the job ID + * @param authentication the authenticated user + * @return job status with progress information + */ + @GetMapping("/jobs/{jobId}/status") + public ResponseEntity getJobStatus( + @PathVariable UUID jobId, + Authentication authentication + ) { + try { + String username = authentication.getName(); + User user = userRepository.findByUsername(username) + .orElseThrow(() -> new IllegalArgumentException("User not found: " + username)); + + // Get job and verify ownership + BatchImportJob job = batchImportJobRepository.findByIdAndUserId(jobId, user.getId()) + .orElseThrow(() -> new IllegalArgumentException("Batch import job not found or access denied")); + + BatchImportJobStatusDTO status = mapJobToStatusDTO(job); + + return ResponseEntity.ok(status); + + } catch (IllegalArgumentException e) { + log.warn("Invalid job status request: {}", e.getMessage()); + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(new ErrorResponse(e.getMessage())); + } catch (Exception e) { + log.error("Failed to get job status", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new ErrorResponse("Failed to get job status: " + e.getMessage())); + } + } + + /** + * Gets detailed file results for a batch import job. + * Called when job completes to show success/failure details. + * + * GET /api/batch-import/jobs/{jobId}/files + * + * @param jobId the job ID + * @param authentication the authenticated user + * @return list of file processing results + */ + @GetMapping("/jobs/{jobId}/files") + public ResponseEntity getJobFileResults( + @PathVariable UUID jobId, + Authentication authentication + ) { + try { + String username = authentication.getName(); + User user = userRepository.findByUsername(username) + .orElseThrow(() -> new IllegalArgumentException("User not found: " + username)); + + // Get job and verify ownership + BatchImportJob job = batchImportJobRepository.findByIdAndUserId(jobId, user.getId()) + .orElseThrow(() -> new IllegalArgumentException("Batch import job not found or access denied")); + + // Get file results + List fileResults = batchImportFileResultRepository + .findByJobIdOrderByProcessedAtDesc(jobId); + + List resultDTOs = fileResults.stream() + .map(this::mapFileResultToDTO) + .toList(); + + return ResponseEntity.ok(resultDTOs); + + } catch (IllegalArgumentException e) { + log.warn("Invalid file results request: {}", e.getMessage()); + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(new ErrorResponse(e.getMessage())); + } catch (Exception e) { + log.error("Failed to get file results", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new ErrorResponse("Failed to get file results: " + e.getMessage())); + } + } + + /** + * Lists recent batch import jobs for the authenticated user. + * + * GET /api/batch-import/jobs?page=0&size=10 + * + * @param page page number (default 0) + * @param size page size (default 10) + * @param authentication the authenticated user + * @return paginated list of batch import jobs + */ + @GetMapping("/jobs") + public ResponseEntity listUserJobs( + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "10") int size, + Authentication authentication + ) { + try { + String username = authentication.getName(); + User user = userRepository.findByUsername(username) + .orElseThrow(() -> new IllegalArgumentException("User not found: " + username)); + + Pageable pageable = PageRequest.of(page, size); + Page jobs = batchImportJobRepository.findByUserIdOrderByCreatedAtDesc( + user.getId(), pageable); + + Page jobDTOs = jobs.map(this::mapJobToStatusDTO); + + return ResponseEntity.ok(jobDTOs); + + } catch (IllegalArgumentException e) { + log.warn("Invalid list jobs request: {}", e.getMessage()); + return ResponseEntity.badRequest().body(new ErrorResponse(e.getMessage())); + } catch (Exception e) { + log.error("Failed to list jobs", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(new ErrorResponse("Failed to list jobs: " + e.getMessage())); + } + } + + /** + * Maps BatchImportJob entity to DTO. + */ + private BatchImportJobStatusDTO mapJobToStatusDTO(BatchImportJob job) { + return new BatchImportJobStatusDTO( + job.getId(), + job.getFilename(), + job.getStatus().name(), + job.getTotalFiles(), + job.getProcessedFiles(), + job.getSuccessCount(), + job.getFailedCount(), + job.getSkippedCount(), + job.getProgressPercentage(), + job.getCreatedAt(), + job.getStartedAt(), + job.getCompletedAt(), + job.getErrorMessage() + ); + } + + /** + * Maps BatchImportFileResult entity to DTO. + */ + private BatchImportFileResultDTO mapFileResultToDTO(BatchImportFileResult result) { + return new BatchImportFileResultDTO( + result.getId(), + result.getFilename(), + result.getFileSize(), + result.getStatus().name(), + result.getActivityId(), + result.getErrorMessage(), + result.getErrorType(), + result.getProcessedAt() + ); + } + + /** + * DTO for batch import job status. + */ + public record BatchImportJobStatusDTO( + UUID id, + String filename, + String status, + Integer totalFiles, + Integer processedFiles, + Integer successCount, + Integer failedCount, + Integer skippedCount, + Integer progressPercentage, + LocalDateTime createdAt, + LocalDateTime startedAt, + LocalDateTime completedAt, + String errorMessage + ) { + } + + /** + * DTO for batch import file result. + */ + public record BatchImportFileResultDTO( + UUID id, + String filename, + Long fileSize, + String status, + UUID activityId, + String errorMessage, + String errorType, + LocalDateTime processedAt + ) { + } + + /** + * DTO for error responses. + */ + public record ErrorResponse(String error) { + } +} diff --git a/src/main/java/org/operaton/fitpub/controller/BatchImportViewController.java b/src/main/java/org/operaton/fitpub/controller/BatchImportViewController.java new file mode 100644 index 0000000..41c8ab8 --- /dev/null +++ b/src/main/java/org/operaton/fitpub/controller/BatchImportViewController.java @@ -0,0 +1,25 @@ +package org.operaton.fitpub.controller; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; + +/** + * View controller for batch import pages. + * Serves Thymeleaf templates for the batch import UI. + */ +@Controller +public class BatchImportViewController { + + /** + * Displays the batch upload page where users can upload ZIP files + * containing multiple activity files for batch processing. + * + * GET /batch-upload + * + * @return the batch upload view template + */ + @GetMapping("/batch-upload") + public String batchUploadPage() { + return "activities/batch-upload"; + } +} diff --git a/src/main/java/org/operaton/fitpub/controller/TimelineController.java b/src/main/java/org/operaton/fitpub/controller/TimelineController.java index 1fef6d8..4d1463d 100644 --- a/src/main/java/org/operaton/fitpub/controller/TimelineController.java +++ b/src/main/java/org/operaton/fitpub/controller/TimelineController.java @@ -9,6 +9,7 @@ import org.operaton.fitpub.service.TimelineService; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.security.core.userdetails.UserDetails; @@ -63,7 +64,8 @@ public class TimelineController { UUID userId = getUserId(userDetails); log.debug("Federated timeline request from user: {}", userId); - Pageable pageable = PageRequest.of(page, size); + // Sort by activity start date descending (latest first) + Pageable pageable = PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "startedAt")); Page timeline = timelineService.getFederatedTimeline(userId, pageable); return ResponseEntity.ok(timeline); @@ -95,7 +97,8 @@ public class TimelineController { log.debug("Public timeline request (unauthenticated)"); } - Pageable pageable = PageRequest.of(page, size); + // Sort by activity start date descending (latest first) + Pageable pageable = PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "startedAt")); Page timeline = timelineService.getPublicTimeline(userId, pageable); return ResponseEntity.ok(timeline); @@ -121,7 +124,8 @@ public class TimelineController { UUID userId = getUserId(userDetails); log.debug("User timeline request from user: {}", userId); - Pageable pageable = PageRequest.of(page, size); + // Sort by activity start date descending (latest first) + Pageable pageable = PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "startedAt")); Page timeline = timelineService.getUserTimeline(userId, pageable); return ResponseEntity.ok(timeline); diff --git a/src/main/java/org/operaton/fitpub/model/entity/BatchImportFileResult.java b/src/main/java/org/operaton/fitpub/model/entity/BatchImportFileResult.java new file mode 100644 index 0000000..9819c4a --- /dev/null +++ b/src/main/java/org/operaton/fitpub/model/entity/BatchImportFileResult.java @@ -0,0 +1,131 @@ +package org.operaton.fitpub.model.entity; + +import jakarta.persistence.*; +import lombok.*; + +import java.time.LocalDateTime; +import java.util.UUID; + +/** + * Entity representing the processing result of an individual file within a batch import job. + * Tracks success/failure status, associated activity, and error details if applicable. + */ +@Entity +@Table(name = "batch_import_file_results") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class BatchImportFileResult { + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + private UUID id; + + @Column(name = "job_id", nullable = false) + private UUID jobId; + + @Column(name = "filename", nullable = false, length = 500) + private String filename; + + @Column(name = "file_size") + private Long fileSize; + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 50) + private FileStatus status; + + @Column(name = "activity_id") + private UUID activityId; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + @Column(name = "error_type", length = 100) + private String errorType; + + @Column(name = "processed_at") + private LocalDateTime processedAt; + + /** + * Status of an individual file processing within a batch import. + */ + public enum FileStatus { + /** File is queued for processing */ + PENDING, + /** File is currently being processed */ + PROCESSING, + /** File was successfully processed and activity created */ + SUCCESS, + /** File processing failed due to an error */ + FAILED, + /** File was skipped (e.g., unsupported format) */ + SKIPPED + } + + /** + * Error type categories for failed file processing. + */ + public static class ErrorType { + public static final String VALIDATION_ERROR = "VALIDATION_ERROR"; + public static final String PARSING_ERROR = "PARSING_ERROR"; + public static final String IO_ERROR = "IO_ERROR"; + public static final String UNSUPPORTED_FORMAT = "UNSUPPORTED_FORMAT"; + public static final String DATABASE_ERROR = "DATABASE_ERROR"; + public static final String UNKNOWN_ERROR = "UNKNOWN_ERROR"; + + private ErrorType() { + // Utility class + } + } + + /** + * Checks if the file was successfully processed. + * + * @return true if status is SUCCESS + */ + public boolean isSuccess() { + return status == FileStatus.SUCCESS; + } + + /** + * Checks if the file processing failed. + * + * @return true if status is FAILED + */ + public boolean isFailed() { + return status == FileStatus.FAILED; + } + + /** + * Checks if the file was skipped. + * + * @return true if status is SKIPPED + */ + public boolean isSkipped() { + return status == FileStatus.SKIPPED; + } + + /** + * Sets default values before persisting. + */ + @PrePersist + protected void onCreate() { + if (status == null) { + status = FileStatus.PENDING; + } + } + + @Override + public String toString() { + return "BatchImportFileResult{" + + "id=" + id + + ", jobId=" + jobId + + ", filename='" + filename + '\'' + + ", status=" + status + + ", activityId=" + activityId + + ", errorType='" + errorType + '\'' + + '}'; + } +} diff --git a/src/main/java/org/operaton/fitpub/model/entity/BatchImportJob.java b/src/main/java/org/operaton/fitpub/model/entity/BatchImportJob.java new file mode 100644 index 0000000..2564a9a --- /dev/null +++ b/src/main/java/org/operaton/fitpub/model/entity/BatchImportJob.java @@ -0,0 +1,167 @@ +package org.operaton.fitpub.model.entity; + +import jakarta.persistence.*; +import lombok.*; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Entity representing a batch import job for processing ZIP files containing multiple activity files. + * Tracks overall progress, status, and results of the batch operation. + */ +@Entity +@Table(name = "batch_import_jobs") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class BatchImportJob { + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + private UUID id; + + @Column(name = "user_id", nullable = false) + private UUID userId; + + @Column(name = "filename", nullable = false, length = 500) + private String filename; + + @Column(name = "total_files", nullable = false) + private Integer totalFiles; + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 50) + private JobStatus status; + + @Column(name = "processed_files", nullable = false) + @Builder.Default + private Integer processedFiles = 0; + + @Column(name = "success_count", nullable = false) + @Builder.Default + private Integer successCount = 0; + + @Column(name = "failed_count", nullable = false) + @Builder.Default + private Integer failedCount = 0; + + @Column(name = "skipped_count", nullable = false) + @Builder.Default + private Integer skippedCount = 0; + + @Column(name = "created_at", nullable = false) + private LocalDateTime createdAt; + + @Column(name = "started_at") + private LocalDateTime startedAt; + + @Column(name = "completed_at") + private LocalDateTime completedAt; + + @Column(name = "error_message", columnDefinition = "TEXT") + private String errorMessage; + + @Column(name = "skip_federation", nullable = false) + @Builder.Default + private Boolean skipFederation = true; + + @OneToMany(mappedBy = "jobId", cascade = CascadeType.ALL, orphanRemoval = true, fetch = FetchType.LAZY) + @Builder.Default + private List fileResults = new ArrayList<>(); + + /** + * Status of the batch import job. + */ + public enum JobStatus { + /** Job created but not yet started */ + PENDING, + /** Job is currently being processed */ + PROCESSING, + /** Job completed successfully (all files processed, some may have failed) */ + COMPLETED, + /** Job failed catastrophically (e.g., ZIP corruption, database error) */ + FAILED, + /** Job was cancelled by user or system */ + CANCELLED + } + + /** + * Calculates the progress percentage of the job. + * + * @return progress as a value between 0 and 100 + */ + public int getProgressPercentage() { + if (totalFiles == null || totalFiles == 0) { + return 0; + } + if (processedFiles == null) { + return 0; + } + return (int) Math.round((processedFiles * 100.0) / totalFiles); + } + + /** + * Checks if the job is in a terminal state (completed, failed, or cancelled). + * + * @return true if job is finished + */ + public boolean isFinished() { + return status == JobStatus.COMPLETED || status == JobStatus.FAILED || status == JobStatus.CANCELLED; + } + + /** + * Checks if the job is currently being processed. + * + * @return true if job is in processing state + */ + public boolean isProcessing() { + return status == JobStatus.PROCESSING; + } + + /** + * Sets default values before persisting. + */ + @PrePersist + protected void onCreate() { + if (createdAt == null) { + createdAt = LocalDateTime.now(); + } + if (status == null) { + status = JobStatus.PENDING; + } + if (processedFiles == null) { + processedFiles = 0; + } + if (successCount == null) { + successCount = 0; + } + if (failedCount == null) { + failedCount = 0; + } + if (skippedCount == null) { + skippedCount = 0; + } + if (skipFederation == null) { + skipFederation = true; + } + } + + @Override + public String toString() { + return "BatchImportJob{" + + "id=" + id + + ", userId=" + userId + + ", filename='" + filename + '\'' + + ", status=" + status + + ", progress=" + getProgressPercentage() + "%" + + ", processed=" + processedFiles + "/" + totalFiles + + ", success=" + successCount + + ", failed=" + failedCount + + '}'; + } +} diff --git a/src/main/java/org/operaton/fitpub/repository/BatchImportFileResultRepository.java b/src/main/java/org/operaton/fitpub/repository/BatchImportFileResultRepository.java new file mode 100644 index 0000000..1256c72 --- /dev/null +++ b/src/main/java/org/operaton/fitpub/repository/BatchImportFileResultRepository.java @@ -0,0 +1,59 @@ +package org.operaton.fitpub.repository; + +import org.operaton.fitpub.model.entity.BatchImportFileResult; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.UUID; + +/** + * Repository for managing batch import file result entities. + * Provides queries for retrieving file processing results within batch import jobs. + */ +@Repository +public interface BatchImportFileResultRepository extends JpaRepository { + + /** + * Finds all file results for a specific batch import job, ordered by processed date (newest first). + * + * @param jobId the batch import job ID + * @return list of file results + */ + List findByJobIdOrderByProcessedAtDesc(UUID jobId); + + /** + * Finds all file results for a specific batch import job with a specific status. + * + * @param jobId the batch import job ID + * @param status the file processing status + * @return list of file results with that status + */ + List findByJobIdAndStatus(UUID jobId, BatchImportFileResult.FileStatus status); + + /** + * Counts the number of file results for a job with a specific status. + * + * @param jobId the batch import job ID + * @param status the file processing status + * @return count of files with that status + */ + long countByJobIdAndStatus(UUID jobId, BatchImportFileResult.FileStatus status); + + /** + * Finds all file results for a specific batch import job, ordered by filename. + * Useful for displaying results in alphabetical order. + * + * @param jobId the batch import job ID + * @return list of file results ordered by filename + */ + List findByJobIdOrderByFilenameAsc(UUID jobId); + + /** + * Deletes all file results for a specific batch import job. + * Typically handled by CASCADE DELETE, but provided for explicit cleanup if needed. + * + * @param jobId the batch import job ID + */ + void deleteByJobId(UUID jobId); +} diff --git a/src/main/java/org/operaton/fitpub/repository/BatchImportJobRepository.java b/src/main/java/org/operaton/fitpub/repository/BatchImportJobRepository.java new file mode 100644 index 0000000..890e918 --- /dev/null +++ b/src/main/java/org/operaton/fitpub/repository/BatchImportJobRepository.java @@ -0,0 +1,76 @@ +package org.operaton.fitpub.repository; + +import org.operaton.fitpub.model.entity.BatchImportJob; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +/** + * Repository for managing batch import job entities. + * Provides queries for job tracking, cleanup, and user-specific job retrieval. + */ +@Repository +public interface BatchImportJobRepository extends JpaRepository { + + /** + * Finds all batch import jobs for a specific user, ordered by creation date (newest first). + * + * @param userId the user ID + * @param pageable pagination information + * @return page of batch import jobs + */ + Page findByUserIdOrderByCreatedAtDesc(UUID userId, Pageable pageable); + + /** + * Finds a batch import job by ID and user ID (for authorization checks). + * + * @param id the job ID + * @param userId the user ID + * @return optional batch import job + */ + Optional findByIdAndUserId(UUID id, UUID userId); + + /** + * Finds all batch import jobs created before a specific date. + * Used for cleanup operations (e.g., deleting jobs older than N days). + * + * @param cutoffDate the cutoff date + * @return list of old batch import jobs + */ + List findByCreatedAtBefore(LocalDateTime cutoffDate); + + /** + * Finds stalled batch import jobs that have been in PROCESSING state for too long. + * A job is considered stalled if it's been processing for more than the timeout period. + * + * @param timeout the timeout threshold (jobs started before this are considered stalled) + * @return list of stalled jobs + */ + @Query("SELECT j FROM BatchImportJob j WHERE j.status = 'PROCESSING' AND j.startedAt < :timeout") + List findStalledJobs(@Param("timeout") LocalDateTime timeout); + + /** + * Counts the number of batch import jobs for a user in a specific status. + * + * @param userId the user ID + * @param status the job status + * @return count of jobs + */ + long countByUserIdAndStatus(UUID userId, BatchImportJob.JobStatus status); + + /** + * Finds all batch import jobs in a specific status. + * + * @param status the job status + * @return list of jobs in that status + */ + List findByStatus(BatchImportJob.JobStatus status); +} diff --git a/src/main/java/org/operaton/fitpub/repository/UserHeatmapGridRepository.java b/src/main/java/org/operaton/fitpub/repository/UserHeatmapGridRepository.java index 975b94a..fc94358 100644 --- a/src/main/java/org/operaton/fitpub/repository/UserHeatmapGridRepository.java +++ b/src/main/java/org/operaton/fitpub/repository/UserHeatmapGridRepository.java @@ -61,11 +61,12 @@ public interface UserHeatmapGridRepository extends JpaRepository 0) { + log.info("Batch import cleanup completed. Deleted {} jobs", deletedCount); + } else { + log.info("Batch import cleanup completed. No jobs to delete"); + } + + } catch (Exception e) { + log.error("Batch import cleanup failed", e); + } + } +} diff --git a/src/main/java/org/operaton/fitpub/service/ActivityFileService.java b/src/main/java/org/operaton/fitpub/service/ActivityFileService.java index 76bf0e5..8466ec5 100644 --- a/src/main/java/org/operaton/fitpub/service/ActivityFileService.java +++ b/src/main/java/org/operaton/fitpub/service/ActivityFileService.java @@ -38,6 +38,59 @@ public class ActivityFileService { private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(new PrecisionModel(), WGS84_SRID); + /** + * Processing options to control which side effects are executed after activity creation. + * Used to skip expensive operations during batch imports and re-execute them later as a batch. + */ + @lombok.Getter + @lombok.Builder + public static class ProcessingOptions { + @lombok.Builder.Default + private final boolean skipPersonalRecords = false; + + @lombok.Builder.Default + private final boolean skipAchievements = false; + + @lombok.Builder.Default + private final boolean skipHeatmap = false; + + @lombok.Builder.Default + private final boolean skipTrainingLoad = false; + + @lombok.Builder.Default + private final boolean skipSummaries = false; + + @lombok.Builder.Default + private final boolean skipWeather = false; + + /** + * Creates options for batch import mode - skips all side effects. + * Analytics and social features are recalculated in a batch after import completes. + * + * @return processing options with all side effects skipped + */ + public static ProcessingOptions batchImportMode() { + return ProcessingOptions.builder() + .skipPersonalRecords(true) + .skipAchievements(true) + .skipHeatmap(true) + .skipTrainingLoad(true) + .skipSummaries(true) + .skipWeather(true) + .build(); + } + + /** + * Creates options for normal mode - executes all side effects. + * This is the default behavior for single activity uploads. + * + * @return processing options with no side effects skipped + */ + public static ProcessingOptions normalMode() { + return ProcessingOptions.builder().build(); + } + } + private final FitFileValidator fitValidator; private final GpxFileValidator gpxValidator; private final FitParser fitParser; @@ -54,6 +107,7 @@ public class ActivityFileService { /** * Processes an uploaded activity file (FIT or GPX) and creates an activity. + * Uses normal processing mode with all side effects enabled. * * @param file the uploaded file * @param userId the user ID @@ -72,6 +126,33 @@ public class ActivityFileService { String title, String description, Activity.Visibility visibility + ) { + return processActivityFile(file, userId, title, description, visibility, ProcessingOptions.normalMode()); + } + + /** + * Processes an uploaded activity file (FIT or GPX) and creates an activity with custom processing options. + * Allows selective skipping of side effects for batch import scenarios. + * + * @param file the uploaded file + * @param userId the user ID + * @param title optional custom title (will be auto-generated if null) + * @param description optional description + * @param visibility visibility level + * @param options processing options to control side effects + * @return the created activity + * @throws FitFileProcessingException if FIT processing fails + * @throws GpxFileProcessingException if GPX processing fails + * @throws UnsupportedFileFormatException if file format is unknown + */ + @Transactional + public Activity processActivityFile( + MultipartFile file, + UUID userId, + String title, + String description, + Activity.Visibility visibility, + ProcessingOptions options ) { try { byte[] fileData = file.getBytes(); @@ -98,7 +179,7 @@ public class ActivityFileService { } // Common processing (same for both formats) - return createActivityFromParsedData(parsedData, userId, title, description, visibility, fileData); + return createActivityFromParsedData(parsedData, userId, title, description, visibility, fileData, options); } catch (IOException e) { throw new RuntimeException("Failed to read activity file", e); } @@ -142,6 +223,8 @@ public class ActivityFileService { /** * Creates an activity from parsed data (internal method). * This method contains all the common logic for creating activities from any format. + * + * @param options processing options to control which side effects are executed */ private Activity createActivityFromParsedData( ParsedActivityData parsedData, @@ -149,7 +232,8 @@ public class ActivityFileService { String title, String description, Activity.Visibility visibility, - byte[] rawFile + byte[] rawFile, + ProcessingOptions options ) { // Generate title if not provided String activityTitle = title != null && !title.isBlank() @@ -204,23 +288,55 @@ public class ActivityFileService { parsedData.getTrackPoints().size(), simplifiedTrack.getNumPoints()); - // Check for personal records and achievements - personalRecordService.checkAndUpdatePersonalRecords(savedActivity); - achievementService.checkAndAwardAchievements(savedActivity); + // Execute side effects based on processing options + // In batch import mode, these are skipped and executed later as a batch - // Update heatmap grid - heatmapGridService.updateHeatmapForActivity(savedActivity); + if (!options.isSkipPersonalRecords()) { + log.debug("Checking personal records for activity {}", savedActivity.getId()); + personalRecordService.checkAndUpdatePersonalRecords(savedActivity); + } else { + log.debug("Skipping personal records check for activity {} (batch mode)", savedActivity.getId()); + } - // Update training load and summaries (async) - trainingLoadService.updateTrainingLoad(savedActivity); - activitySummaryService.updateSummariesForActivity(savedActivity); + if (!options.isSkipAchievements()) { + log.debug("Checking achievements for activity {}", savedActivity.getId()); + achievementService.checkAndAwardAchievements(savedActivity); + } else { + log.debug("Skipping achievements check for activity {} (batch mode)", savedActivity.getId()); + } - // Fetch weather data (async, non-blocking) - try { - weatherService.fetchWeatherForActivity(savedActivity); - } catch (Exception e) { - log.warn("Failed to fetch weather data for activity {}: {}", savedActivity.getId(), e.getMessage()); - // Don't fail the activity creation if weather fetching fails + if (!options.isSkipHeatmap()) { + log.debug("Updating heatmap for activity {}", savedActivity.getId()); + heatmapGridService.updateHeatmapForActivity(savedActivity); + } else { + log.debug("Skipping heatmap update for activity {} (batch mode)", savedActivity.getId()); + } + + if (!options.isSkipTrainingLoad()) { + log.debug("Updating training load for activity {}", savedActivity.getId()); + trainingLoadService.updateTrainingLoad(savedActivity); + } else { + log.debug("Skipping training load update for activity {} (batch mode)", savedActivity.getId()); + } + + if (!options.isSkipSummaries()) { + log.debug("Updating summaries for activity {}", savedActivity.getId()); + activitySummaryService.updateSummariesForActivity(savedActivity); + } else { + log.debug("Skipping summaries update for activity {} (batch mode)", savedActivity.getId()); + } + + if (!options.isSkipWeather()) { + // Fetch weather data (async, non-blocking) + try { + log.debug("Fetching weather for activity {}", savedActivity.getId()); + weatherService.fetchWeatherForActivity(savedActivity); + } catch (Exception e) { + log.warn("Failed to fetch weather data for activity {}: {}", savedActivity.getId(), e.getMessage()); + // Don't fail the activity creation if weather fetching fails + } + } else { + log.debug("Skipping weather fetch for activity {} (batch mode)", savedActivity.getId()); } return savedActivity; diff --git a/src/main/java/org/operaton/fitpub/service/BatchImportService.java b/src/main/java/org/operaton/fitpub/service/BatchImportService.java new file mode 100644 index 0000000..f009b1c --- /dev/null +++ b/src/main/java/org/operaton/fitpub/service/BatchImportService.java @@ -0,0 +1,597 @@ +package org.operaton.fitpub.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.operaton.fitpub.model.entity.Activity; +import org.operaton.fitpub.model.entity.BatchImportFileResult; +import org.operaton.fitpub.model.entity.BatchImportJob; +import org.operaton.fitpub.model.entity.User; +import org.operaton.fitpub.repository.ActivityRepository; +import org.operaton.fitpub.repository.BatchImportFileResultRepository; +import org.operaton.fitpub.repository.BatchImportJobRepository; +import org.operaton.fitpub.repository.UserRepository; +import org.operaton.fitpub.util.ByteArrayMultipartFile; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Service for managing batch imports of activity files from ZIP archives. + * Handles asynchronous processing, progress tracking, and analytics recalculation. + */ +@Service +@Slf4j +public class BatchImportService { + + // Validation constants + private static final long MAX_ZIP_SIZE = 500L * 1024 * 1024; // 500 MB + private static final int MAX_FILES_IN_ZIP = 1000; + private static final long MAX_INDIVIDUAL_FILE_SIZE = 50L * 1024 * 1024; // 50 MB + + private final BatchImportJobRepository batchImportJobRepository; + private final BatchImportFileResultRepository batchImportFileResultRepository; + private final ActivityFileService activityFileService; + private final ActivityRepository activityRepository; + private final UserRepository userRepository; + private final PersonalRecordService personalRecordService; + private final AchievementService achievementService; + private final HeatmapGridService heatmapGridService; + private final TrainingLoadService trainingLoadService; + private final ActivitySummaryService activitySummaryService; + private final BatchImportService self; + + public BatchImportService( + BatchImportJobRepository batchImportJobRepository, + BatchImportFileResultRepository batchImportFileResultRepository, + ActivityFileService activityFileService, + ActivityRepository activityRepository, + UserRepository userRepository, + PersonalRecordService personalRecordService, + AchievementService achievementService, + HeatmapGridService heatmapGridService, + TrainingLoadService trainingLoadService, + ActivitySummaryService activitySummaryService, + @org.springframework.context.annotation.Lazy BatchImportService self) { + this.batchImportJobRepository = batchImportJobRepository; + this.batchImportFileResultRepository = batchImportFileResultRepository; + this.activityFileService = activityFileService; + this.activityRepository = activityRepository; + this.userRepository = userRepository; + this.personalRecordService = personalRecordService; + this.achievementService = achievementService; + this.heatmapGridService = heatmapGridService; + this.trainingLoadService = trainingLoadService; + this.activitySummaryService = activitySummaryService; + this.self = self; + } + + /** + * Creates a batch import job from an uploaded ZIP file. + * Validates the ZIP, extracts file list, creates job and file result entities. + * + * @param zipFile the uploaded ZIP file + * @param userId the user ID + * @return the created batch import job + * @throws IllegalArgumentException if validation fails + */ + @Transactional + public BatchImportJob createBatchImportJob(MultipartFile zipFile, UUID userId) { + log.info("Creating batch import job for user {} with file {}", userId, zipFile.getOriginalFilename()); + + // Validate ZIP file + validateZipFile(zipFile); + + // Extract file list from ZIP + List fileEntries; + try { + fileEntries = extractFileList(zipFile.getBytes()); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to read ZIP file: " + e.getMessage(), e); + } + + if (fileEntries.isEmpty()) { + throw new IllegalArgumentException("ZIP file contains no valid activity files (.fit or .gpx)"); + } + + if (fileEntries.size() > MAX_FILES_IN_ZIP) { + throw new IllegalArgumentException( + String.format("ZIP file contains too many files (%d). Maximum allowed: %d", + fileEntries.size(), MAX_FILES_IN_ZIP) + ); + } + + // Create batch import job + BatchImportJob job = BatchImportJob.builder() + .userId(userId) + .filename(zipFile.getOriginalFilename()) + .totalFiles(fileEntries.size()) + .status(BatchImportJob.JobStatus.PENDING) + .skipFederation(true) + .build(); + + job = batchImportJobRepository.save(job); + + // Create file result entries + for (FileEntry entry : fileEntries) { + BatchImportFileResult fileResult = BatchImportFileResult.builder() + .jobId(job.getId()) + .filename(entry.getName()) + .fileSize(entry.getSize()) + .status(BatchImportFileResult.FileStatus.PENDING) + .build(); + + batchImportFileResultRepository.save(fileResult); + } + + log.info("Created batch import job {} with {} files", job.getId(), fileEntries.size()); + + // Schedule async processing AFTER transaction commits to ensure job is visible in database + final UUID jobId = job.getId(); + final byte[] zipBytes; + try { + zipBytes = zipFile.getBytes(); + } catch (IOException e) { + log.error("Failed to read ZIP file bytes", e); + markJobAsFailed(job.getId(), "Failed to read ZIP file: " + e.getMessage()); + throw new RuntimeException("Failed to read ZIP file", e); + } + + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + log.info("Transaction committed, starting async processing for job {}", jobId); + try { + self.processBatchImportAsync(jobId, zipBytes); + log.info("Async processing started for job {}", jobId); + } catch (Exception e) { + log.error("Failed to start async processing for job {}", jobId, e); + // Job will remain in PENDING state, user can retry + } + } + }); + + log.info("Registered async processing callback for job {} (will start after commit)", jobId); + return job; + } + + /** + * Processes a batch import job asynchronously. + * Runs in a background thread pool and processes files one by one. + * + * @param jobId the batch import job ID + * @param zipData the ZIP file data + */ + @Async("batchImportExecutor") + public void processBatchImportAsync(UUID jobId, byte[] zipData) { + log.info("Starting async processing for batch import job {}", jobId); + + try { + // Mark job as processing + markJobAsProcessing(jobId); + + // Extract files from ZIP + List fileEntries = extractFileList(zipData); + + // Get job and file results + BatchImportJob job = batchImportJobRepository.findById(jobId) + .orElseThrow(() -> new IllegalStateException("Batch import job not found: " + jobId)); + + List fileResults = batchImportFileResultRepository.findByJobIdOrderByFilenameAsc(jobId); + + // Process each file + for (int i = 0; i < fileEntries.size(); i++) { + FileEntry entry = fileEntries.get(i); + BatchImportFileResult fileResult = fileResults.stream() + .filter(fr -> fr.getFilename().equals(entry.getName())) + .findFirst() + .orElse(null); + + if (fileResult == null) { + log.warn("File result not found for {}, skipping", entry.getName()); + continue; + } + + log.info("Processing file {}/{}: {} ({} bytes)", + i + 1, fileEntries.size(), entry.getName(), entry.getSize()); + + processIndividualFile(job, entry, fileResult); + } + + // Phase 2: Batch recalculate analytics + log.info("Phase 2: Recalculating analytics for batch import job {}", jobId); + recalculateAnalyticsForJob(job); + + // Mark job as completed + markJobAsCompleted(jobId); + + log.info("Batch import job {} completed successfully. Processed: {}, Success: {}, Failed: {}", + jobId, job.getProcessedFiles(), job.getSuccessCount(), job.getFailedCount()); + + } catch (Exception e) { + log.error("Batch import job {} failed with error", jobId, e); + markJobAsFailed(jobId, e.getMessage()); + } + } + + /** + * Processes an individual file within a batch import. + * Uses REQUIRES_NEW transaction for fault isolation. + * + * @param job the batch import job + * @param entry the file entry + * @param fileResult the file result entity + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + protected void processIndividualFile(BatchImportJob job, FileEntry entry, BatchImportFileResult fileResult) { + try { + // Mark file as processing + fileResult.setStatus(BatchImportFileResult.FileStatus.PROCESSING); + batchImportFileResultRepository.save(fileResult); + + // Create MultipartFile from file data + ByteArrayMultipartFile file = new ByteArrayMultipartFile( + "file", + entry.getName(), + "application/octet-stream", + entry.getData() + ); + + // Process activity file with batch import mode (skip all side effects) + Activity activity = activityFileService.processActivityFile( + file, + job.getUserId(), + null, // Auto-generate title + null, // No description + Activity.Visibility.PUBLIC, + ActivityFileService.ProcessingOptions.batchImportMode() + ); + + // Mark file as success + fileResult.setStatus(BatchImportFileResult.FileStatus.SUCCESS); + fileResult.setActivityId(activity.getId()); + fileResult.setProcessedAt(LocalDateTime.now()); + batchImportFileResultRepository.save(fileResult); + + // Increment job progress + incrementJobProgress(job.getId(), true, false, false); + + log.debug("Successfully processed file {} -> activity {}", entry.getName(), activity.getId()); + + } catch (Exception e) { + log.warn("Failed to process file {}: {}", entry.getName(), e.getMessage()); + + // Determine error type + String errorType = determineErrorType(e); + + // Mark file as failed + fileResult.setStatus(BatchImportFileResult.FileStatus.FAILED); + fileResult.setErrorMessage(e.getMessage()); + fileResult.setErrorType(errorType); + fileResult.setProcessedAt(LocalDateTime.now()); + batchImportFileResultRepository.save(fileResult); + + // Increment job progress + incrementJobProgress(job.getId(), false, true, false); + } + } + + /** + * Recalculates analytics for all activities in a batch import job. + * This is Phase 2 of the batch import process. + * + * @param job the batch import job + */ + @Transactional + protected void recalculateAnalyticsForJob(BatchImportJob job) { + log.info("Recalculating analytics for batch import job {} (user {})", job.getId(), job.getUserId()); + + try { + // Get all successfully imported activities + List successfulResults = batchImportFileResultRepository + .findByJobIdAndStatus(job.getId(), BatchImportFileResult.FileStatus.SUCCESS); + + List activityIds = successfulResults.stream() + .map(BatchImportFileResult::getActivityId) + .filter(id -> id != null) + .toList(); + + if (activityIds.isEmpty()) { + log.info("No successful activities to process analytics for"); + return; + } + + log.info("Recalculating analytics for {} activities", activityIds.size()); + + // Fetch user for heatmap recalculation + User user = userRepository.findById(job.getUserId()) + .orElseThrow(() -> new IllegalStateException("User not found: " + job.getUserId())); + + // Recalculate heatmap (single full rebuild is more efficient than incremental) + log.debug("Rebuilding user heatmap..."); + heatmapGridService.recalculateUserHeatmap(user); + + // Recalculate personal records for each activity + log.debug("Recalculating personal records..."); + for (UUID activityId : activityIds) { + Activity activity = activityRepository.findById(activityId).orElse(null); + if (activity != null) { + personalRecordService.checkAndUpdatePersonalRecords(activity); + } + } + + // Recalculate achievements for each activity + log.debug("Recalculating achievements..."); + for (UUID activityId : activityIds) { + Activity activity = activityRepository.findById(activityId).orElse(null); + if (activity != null) { + achievementService.checkAndAwardAchievements(activity); + } + } + + // Recalculate training load for each activity + log.debug("Recalculating training load..."); + for (UUID activityId : activityIds) { + Activity activity = activityRepository.findById(activityId).orElse(null); + if (activity != null) { + trainingLoadService.updateTrainingLoad(activity); + } + } + + // Recalculate activity summaries (async) + log.debug("Updating activity summaries..."); + for (UUID activityId : activityIds) { + Activity activity = activityRepository.findById(activityId).orElse(null); + if (activity != null) { + activitySummaryService.updateSummariesForActivity(activity); + } + } + + log.info("Analytics recalculation completed for batch import job {}", job.getId()); + + } catch (Exception e) { + log.error("Failed to recalculate analytics for batch import job {}", job.getId(), e); + // Don't fail the job - analytics can be recalculated manually if needed + } + } + + /** + * Increments job progress counters atomically. + * Uses REQUIRES_NEW transaction to avoid blocking file processing. + * + * @param jobId the job ID + * @param success true if file was successful + * @param failed true if file failed + * @param skipped true if file was skipped + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + protected void incrementJobProgress(UUID jobId, boolean success, boolean failed, boolean skipped) { + BatchImportJob job = batchImportJobRepository.findById(jobId) + .orElseThrow(() -> new IllegalStateException("Batch import job not found: " + jobId)); + + job.setProcessedFiles(job.getProcessedFiles() + 1); + + if (success) { + job.setSuccessCount(job.getSuccessCount() + 1); + } + if (failed) { + job.setFailedCount(job.getFailedCount() + 1); + } + if (skipped) { + job.setSkippedCount(job.getSkippedCount() + 1); + } + + batchImportJobRepository.save(job); + + log.debug("Job {} progress: {}/{} (success: {}, failed: {}, skipped: {})", + jobId, job.getProcessedFiles(), job.getTotalFiles(), + job.getSuccessCount(), job.getFailedCount(), job.getSkippedCount()); + } + + /** + * Marks a job as processing. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + protected void markJobAsProcessing(UUID jobId) { + BatchImportJob job = batchImportJobRepository.findById(jobId) + .orElseThrow(() -> new IllegalStateException("Batch import job not found: " + jobId)); + + job.setStatus(BatchImportJob.JobStatus.PROCESSING); + job.setStartedAt(LocalDateTime.now()); + batchImportJobRepository.save(job); + + log.info("Marked batch import job {} as PROCESSING", jobId); + } + + /** + * Marks a job as completed. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + protected void markJobAsCompleted(UUID jobId) { + BatchImportJob job = batchImportJobRepository.findById(jobId) + .orElseThrow(() -> new IllegalStateException("Batch import job not found: " + jobId)); + + job.setStatus(BatchImportJob.JobStatus.COMPLETED); + job.setCompletedAt(LocalDateTime.now()); + batchImportJobRepository.save(job); + + log.info("Marked batch import job {} as COMPLETED", jobId); + } + + /** + * Marks a job as failed with an error message. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + protected void markJobAsFailed(UUID jobId, String errorMessage) { + BatchImportJob job = batchImportJobRepository.findById(jobId) + .orElseThrow(() -> new IllegalStateException("Batch import job not found: " + jobId)); + + job.setStatus(BatchImportJob.JobStatus.FAILED); + job.setErrorMessage(errorMessage); + job.setCompletedAt(LocalDateTime.now()); + batchImportJobRepository.save(job); + + log.error("Marked batch import job {} as FAILED: {}", jobId, errorMessage); + } + + /** + * Cleans up old batch import jobs older than the specified retention period. + * + * @param retentionDays number of days to keep jobs + * @return number of jobs deleted + */ + @Transactional + public int cleanupOldJobs(int retentionDays) { + LocalDateTime cutoffDate = LocalDateTime.now().minusDays(retentionDays); + List oldJobs = batchImportJobRepository.findByCreatedAtBefore(cutoffDate); + + if (oldJobs.isEmpty()) { + log.info("No batch import jobs older than {} days found", retentionDays); + return 0; + } + + log.info("Found {} batch import jobs older than {} days, deleting...", oldJobs.size(), retentionDays); + + for (BatchImportJob job : oldJobs) { + batchImportJobRepository.delete(job); + log.debug("Deleted batch import job {} (created {})", job.getId(), job.getCreatedAt()); + } + + log.info("Deleted {} old batch import jobs", oldJobs.size()); + return oldJobs.size(); + } + + /** + * Validates ZIP file constraints. + */ + private void validateZipFile(MultipartFile zipFile) { + if (zipFile == null || zipFile.isEmpty()) { + throw new IllegalArgumentException("ZIP file is required"); + } + + if (zipFile.getSize() > MAX_ZIP_SIZE) { + throw new IllegalArgumentException( + String.format("ZIP file is too large (%d MB). Maximum allowed: %d MB", + zipFile.getSize() / (1024 * 1024), + MAX_ZIP_SIZE / (1024 * 1024)) + ); + } + + String filename = zipFile.getOriginalFilename(); + if (filename == null || !filename.toLowerCase().endsWith(".zip")) { + throw new IllegalArgumentException("File must be a ZIP archive (.zip)"); + } + } + + /** + * Extracts file list from ZIP archive. + * Only includes .fit and .gpx files. + */ + private List extractFileList(byte[] zipData) throws IOException { + List fileEntries = new ArrayList<>(); + + try (ByteArrayInputStream bais = new ByteArrayInputStream(zipData); + ZipInputStream zis = new ZipInputStream(bais)) { + + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + // Skip directories + if (entry.isDirectory()) { + continue; + } + + String name = entry.getName(); + // Skip hidden files and __MACOSX files + if (name.startsWith("__MACOSX") || name.contains("/.")) { + continue; + } + + // Extract just the filename (no path) + String filename = name.substring(name.lastIndexOf('/') + 1); + + // Only accept .fit and .gpx files + String lowerFilename = filename.toLowerCase(); + if (!lowerFilename.endsWith(".fit") && !lowerFilename.endsWith(".gpx")) { + log.debug("Skipping non-activity file: {}", filename); + continue; + } + + // Check file size + if (entry.getSize() > MAX_INDIVIDUAL_FILE_SIZE) { + log.warn("Skipping file {} - too large ({} MB, max: {} MB)", + filename, entry.getSize() / (1024 * 1024), + MAX_INDIVIDUAL_FILE_SIZE / (1024 * 1024)); + continue; + } + + // Read file data + byte[] fileData = zis.readAllBytes(); + + fileEntries.add(new FileEntry(filename, fileData.length, fileData)); + } + } + + log.info("Extracted {} valid activity files from ZIP", fileEntries.size()); + return fileEntries; + } + + /** + * Determines error type from exception. + */ + private String determineErrorType(Exception e) { + String className = e.getClass().getSimpleName(); + + if (className.contains("Validation")) { + return BatchImportFileResult.ErrorType.VALIDATION_ERROR; + } else if (className.contains("Parsing") || className.contains("Format")) { + return BatchImportFileResult.ErrorType.PARSING_ERROR; + } else if (className.contains("Unsupported")) { + return BatchImportFileResult.ErrorType.UNSUPPORTED_FORMAT; + } else if (className.contains("IO") || className.contains("File")) { + return BatchImportFileResult.ErrorType.IO_ERROR; + } else if (className.contains("SQL") || className.contains("Database") || className.contains("JPA")) { + return BatchImportFileResult.ErrorType.DATABASE_ERROR; + } else { + return BatchImportFileResult.ErrorType.UNKNOWN_ERROR; + } + } + + /** + * Internal class for file entries extracted from ZIP. + */ + private static class FileEntry { + private final String name; + private final long size; + private final byte[] data; + + public FileEntry(String name, long size, byte[] data) { + this.name = name; + this.size = size; + this.data = data; + } + + public String getName() { + return name; + } + + public long getSize() { + return size; + } + + public byte[] getData() { + return data; + } + } +} diff --git a/src/main/java/org/operaton/fitpub/service/HeatmapGridService.java b/src/main/java/org/operaton/fitpub/service/HeatmapGridService.java index a3ee319..e928097 100644 --- a/src/main/java/org/operaton/fitpub/service/HeatmapGridService.java +++ b/src/main/java/org/operaton/fitpub/service/HeatmapGridService.java @@ -2,6 +2,7 @@ package org.operaton.fitpub.service; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.persistence.EntityManager; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.locationtech.jts.geom.Coordinate; @@ -23,13 +24,29 @@ import java.util.*; * Aggregates GPS track points into spatial grid cells for efficient heatmap rendering. */ @Service -@RequiredArgsConstructor @Slf4j public class HeatmapGridService { private final UserHeatmapGridRepository heatmapGridRepository; private final ActivityRepository activityRepository; private final ObjectMapper objectMapper; + private final EntityManager entityManager; + private final org.springframework.jdbc.core.JdbcTemplate jdbcTemplate; + + // Constructor + public HeatmapGridService( + UserHeatmapGridRepository heatmapGridRepository, + ActivityRepository activityRepository, + ObjectMapper objectMapper, + EntityManager entityManager, + org.springframework.jdbc.core.JdbcTemplate jdbcTemplate + ) { + this.heatmapGridRepository = heatmapGridRepository; + this.activityRepository = activityRepository; + this.objectMapper = objectMapper; + this.entityManager = entityManager; + this.jdbcTemplate = jdbcTemplate; + } /** * Grid resolution in degrees (~100m at equator). @@ -96,8 +113,17 @@ public class HeatmapGridService { public void recalculateUserHeatmap(User user) { log.info("Recalculating heatmap for user {}", user.getUsername()); - // Delete existing grid - heatmapGridRepository.deleteByUserId(user.getId()); + // Delete existing grid using direct JDBC to ensure immediate execution + log.debug("Deleting existing heatmap data for user {} using direct JDBC", user.getId()); + int deletedRows = jdbcTemplate.update( + "DELETE FROM user_heatmap_grid WHERE user_id = ?", + user.getId() + ); + log.info("Deleted {} existing heatmap grid cells for user {}", deletedRows, user.getUsername()); + + // Flush and clear to ensure Hibernate sees the changes + entityManager.flush(); + entityManager.clear(); // Get all activities for user List activities = activityRepository.findByUserIdOrderByStartedAtDesc(user.getId()); @@ -107,28 +133,37 @@ public class HeatmapGridService { } // Aggregate all grid cells across all activities - Map allCellCounts = new HashMap<>(); + // Use WKT (Well-Known Text) as key to ensure exact geometry matching + Map allCellCounts = new HashMap<>(); for (Activity activity : activities) { List gridCells = extractGridCellsFromActivity(activity); for (Point cell : gridCells) { - String key = cellKey(cell); - allCellCounts.put(key, allCellCounts.getOrDefault(key, 0) + 1); + // Use WKT as the key for exact geometry matching + String wktKey = cell.toText(); + GridCellData cellData = allCellCounts.get(wktKey); + if (cellData == null) { + cellData = new GridCellData(cell, 0); + allCellCounts.put(wktKey, cellData); + } + cellData.incrementCount(); } } // Bulk insert grid cells List gridEntities = new ArrayList<>(); - for (Map.Entry entry : allCellCounts.entrySet()) { - Point cell = parseCell(entry.getKey()); + for (GridCellData cellData : allCellCounts.values()) { UserHeatmapGrid grid = UserHeatmapGrid.builder() .userId(user.getId()) - .gridCell(cell) - .pointCount(entry.getValue()) + .gridCell(cellData.getPoint()) + .pointCount(cellData.getCount()) .build(); gridEntities.add(grid); } + log.info("Aggregated {} unique grid cells from {} activities", + allCellCounts.size(), activities.size()); + heatmapGridRepository.saveAll(gridEntities); log.info("Recalculated {} grid cells for user {} from {} activities", gridEntities.size(), user.getUsername(), activities.size()); @@ -274,4 +309,30 @@ public class HeatmapGridService { double lon = Double.parseDouble(parts[1]); return geometryFactory.createPoint(new Coordinate(lon, lat)); } + + /** + * Helper class to store grid cell Point and its count. + * Used during aggregation to avoid recreating Point objects. + */ + private static class GridCellData { + private final Point point; + private int count; + + public GridCellData(Point point, int initialCount) { + this.point = point; + this.count = initialCount; + } + + public void incrementCount() { + this.count++; + } + + public Point getPoint() { + return point; + } + + public int getCount() { + return count; + } + } } diff --git a/src/main/java/org/operaton/fitpub/service/TimelineService.java b/src/main/java/org/operaton/fitpub/service/TimelineService.java index 3d94728..53f4c5a 100644 --- a/src/main/java/org/operaton/fitpub/service/TimelineService.java +++ b/src/main/java/org/operaton/fitpub/service/TimelineService.java @@ -75,7 +75,9 @@ public class TimelineService { // 3. Fetch local activities from followed users (fetch more to account for merging) // We fetch double the page size to have enough items after merging - Pageable expandedPageable = PageRequest.of(0, pageable.getPageSize() * 2); + // Explicitly sort by startedAt DESC (latest first) + Pageable expandedPageable = PageRequest.of(0, pageable.getPageSize() * 2, + org.springframework.data.domain.Sort.by(org.springframework.data.domain.Sort.Direction.DESC, "startedAt")); Page localActivities = activityRepository.findByUserIdInAndVisibilityInOrderByStartedAtDesc( followedUserIds, List.of(Activity.Visibility.PUBLIC, Activity.Visibility.FOLLOWERS), @@ -85,6 +87,7 @@ public class TimelineService { // 4. Fetch remote activities from followed remote actors (if any) List remoteActivities = new ArrayList<>(); if (!remoteActorUris.isEmpty()) { + // Use same pageable with explicit sort for remote activities Page remoteActivitiesPage = remoteActivityRepository.findByRemoteActorUriInAndVisibilityIn( remoteActorUris, List.of(RemoteActivity.Visibility.PUBLIC, RemoteActivity.Visibility.FOLLOWERS), diff --git a/src/main/java/org/operaton/fitpub/util/ByteArrayMultipartFile.java b/src/main/java/org/operaton/fitpub/util/ByteArrayMultipartFile.java new file mode 100644 index 0000000..7113240 --- /dev/null +++ b/src/main/java/org/operaton/fitpub/util/ByteArrayMultipartFile.java @@ -0,0 +1,67 @@ +package org.operaton.fitpub.util; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +/** + * Simple implementation of MultipartFile for in-memory byte arrays. + * Used for batch import where files are extracted from ZIP archives. + */ +public class ByteArrayMultipartFile implements MultipartFile { + + private final String name; + private final String originalFilename; + private final String contentType; + private final byte[] content; + + public ByteArrayMultipartFile(String name, String originalFilename, String contentType, byte[] content) { + this.name = name; + this.originalFilename = originalFilename; + this.contentType = contentType; + this.content = content != null ? content : new byte[0]; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getOriginalFilename() { + return originalFilename; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public boolean isEmpty() { + return content.length == 0; + } + + @Override + public long getSize() { + return content.length; + } + + @Override + public byte[] getBytes() throws IOException { + return content; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(content); + } + + @Override + public void transferTo(File dest) throws IOException, IllegalStateException { + throw new UnsupportedOperationException("transferTo not supported for ByteArrayMultipartFile"); + } +} diff --git a/src/main/resources/db/migration/V16__create_batch_import_tables.sql b/src/main/resources/db/migration/V16__create_batch_import_tables.sql new file mode 100644 index 0000000..bd9e971 --- /dev/null +++ b/src/main/resources/db/migration/V16__create_batch_import_tables.sql @@ -0,0 +1,62 @@ +-- Migration V16: Create batch import tables for asynchronous ZIP file processing +-- Purpose: Support batch import of hundreds of FIT/GPX files with progress tracking + +-- Main job tracking table +CREATE TABLE batch_import_jobs ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(500) NOT NULL, + total_files INTEGER NOT NULL, + status VARCHAR(50) NOT NULL, + processed_files INTEGER NOT NULL DEFAULT 0, + success_count INTEGER NOT NULL DEFAULT 0, + failed_count INTEGER NOT NULL DEFAULT 0, + skipped_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL, + started_at TIMESTAMP, + completed_at TIMESTAMP, + error_message TEXT, + skip_federation BOOLEAN NOT NULL DEFAULT TRUE, + + -- Constraints + CONSTRAINT batch_import_jobs_status_check CHECK (status IN ('PENDING', 'PROCESSING', 'COMPLETED', 'FAILED', 'CANCELLED')), + CONSTRAINT batch_import_jobs_total_files_positive CHECK (total_files > 0), + CONSTRAINT batch_import_jobs_processed_files_non_negative CHECK (processed_files >= 0), + CONSTRAINT batch_import_jobs_counts_non_negative CHECK ( + success_count >= 0 AND + failed_count >= 0 AND + skipped_count >= 0 + ) +); + +-- Individual file result tracking table +CREATE TABLE batch_import_file_results ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES batch_import_jobs(id) ON DELETE CASCADE, + filename VARCHAR(500) NOT NULL, + file_size BIGINT, + status VARCHAR(50) NOT NULL, + activity_id UUID REFERENCES activities(id) ON DELETE SET NULL, + error_message TEXT, + error_type VARCHAR(100), + processed_at TIMESTAMP, + + -- Constraints + CONSTRAINT batch_import_file_results_status_check CHECK (status IN ('PENDING', 'PROCESSING', 'SUCCESS', 'FAILED', 'SKIPPED')), + CONSTRAINT batch_import_file_results_file_size_non_negative CHECK (file_size IS NULL OR file_size >= 0) +); + +-- Indexes for performance +CREATE INDEX idx_batch_import_jobs_user_id ON batch_import_jobs(user_id); +CREATE INDEX idx_batch_import_jobs_created_at ON batch_import_jobs(created_at); +CREATE INDEX idx_batch_import_jobs_status ON batch_import_jobs(status); +CREATE INDEX idx_batch_import_file_results_job_id ON batch_import_file_results(job_id); +CREATE INDEX idx_batch_import_file_results_status ON batch_import_file_results(status); + +-- Comments for documentation +COMMENT ON TABLE batch_import_jobs IS 'Tracks asynchronous batch import jobs for ZIP files containing multiple activity files'; +COMMENT ON TABLE batch_import_file_results IS 'Tracks processing results for individual files within a batch import job'; +COMMENT ON COLUMN batch_import_jobs.skip_federation IS 'When true, imported activities will not trigger ActivityPub federation'; +COMMENT ON COLUMN batch_import_jobs.total_files IS 'Total number of files extracted from ZIP'; +COMMENT ON COLUMN batch_import_jobs.processed_files IS 'Number of files processed (success + failed + skipped)'; +COMMENT ON COLUMN batch_import_file_results.error_type IS 'Category of error (e.g., VALIDATION_ERROR, PARSING_ERROR, IO_ERROR)'; diff --git a/src/main/resources/static/js/auth.js b/src/main/resources/static/js/auth.js index ee60d00..dd774c6 100644 --- a/src/main/resources/static/js/auth.js +++ b/src/main/resources/static/js/auth.js @@ -216,6 +216,7 @@ const FitPubAuth = { const usernameDisplay = document.getElementById('usernameDisplay'); const myActivitiesLink = document.getElementById('myActivitiesLink'); const uploadLink = document.getElementById('uploadLink'); + const batchUploadLink = document.getElementById('batchUploadLink'); const analyticsLink = document.getElementById('analyticsLink'); const heatmapLink = document.getElementById('heatmapLink'); const notificationsBell = document.getElementById('notificationsBell'); @@ -238,6 +239,10 @@ const FitPubAuth = { uploadLink.style.display = ''; uploadLink.parentElement.style.display = ''; } + if (batchUploadLink) { + batchUploadLink.style.display = ''; + batchUploadLink.parentElement.style.display = ''; + } if (analyticsLink) { analyticsLink.style.display = ''; analyticsLink.parentElement.style.display = ''; @@ -281,6 +286,10 @@ const FitPubAuth = { uploadLink.style.display = 'none'; uploadLink.parentElement.style.display = 'none'; } + if (batchUploadLink) { + batchUploadLink.style.display = 'none'; + batchUploadLink.parentElement.style.display = 'none'; + } if (analyticsLink) { analyticsLink.style.display = 'none'; analyticsLink.parentElement.style.display = 'none'; diff --git a/src/main/resources/static/js/batch-import.js b/src/main/resources/static/js/batch-import.js new file mode 100644 index 0000000..fcfbdfa --- /dev/null +++ b/src/main/resources/static/js/batch-import.js @@ -0,0 +1,509 @@ +/** + * Batch Import JavaScript Module + * Handles ZIP file upload, progress tracking, and results display for batch activity imports. + */ + +(function() { + 'use strict'; + + let currentJobId = null; + let pollingInterval = null; + let selectedFile = null; + + // Initialize page + document.addEventListener('DOMContentLoaded', function() { + initializeUploadZone(); + loadRecentJobs(); + checkAuthentication(); + }); + + /** + * Check if user is authenticated, show warning if not. + * Don't redirect immediately - let them browse, but prevent upload. + */ + function checkAuthentication() { + const token = localStorage.getItem('jwtToken'); + if (!token) { + console.log('No auth token found - user needs to login to upload'); + // Show a message but don't redirect yet + const uploadZone = document.getElementById('uploadZone'); + if (uploadZone) { + uploadZone.innerHTML = ` + +

Authentication Required

+

Please log in to use batch import

+ + Login + + `; + } + } + } + + /** + * Initialize drag-and-drop upload zone. + */ + function initializeUploadZone() { + const uploadZone = document.getElementById('uploadZone'); + const fileInput = document.getElementById('zipFileInput'); + const uploadButton = document.getElementById('uploadButton'); + + // Click to browse + uploadZone.addEventListener('click', (e) => { + if (e.target.type !== 'button') { + fileInput.click(); + } + }); + + // File selected via input + fileInput.addEventListener('change', handleFileSelect); + + // Drag and drop + uploadZone.addEventListener('dragover', (e) => { + e.preventDefault(); + uploadZone.classList.add('drag-over'); + }); + + uploadZone.addEventListener('dragleave', () => { + uploadZone.classList.remove('drag-over'); + }); + + uploadZone.addEventListener('drop', (e) => { + e.preventDefault(); + uploadZone.classList.remove('drag-over'); + + const files = e.dataTransfer.files; + if (files.length > 0) { + const file = files[0]; + if (file.name.toLowerCase().endsWith('.zip')) { + selectedFile = file; + displaySelectedFile(file); + } else { + FitPub.showAlert('Please select a ZIP file', 'danger'); + } + } + }); + + // Upload button click + uploadButton.addEventListener('click', uploadZipFile); + } + + /** + * Handle file selection from input. + */ + function handleFileSelect(e) { + const file = e.target.files[0]; + if (file) { + if (file.name.toLowerCase().endsWith('.zip')) { + selectedFile = file; + displaySelectedFile(file); + } else { + FitPub.showAlert('Please select a ZIP file', 'danger'); + e.target.value = ''; + } + } + } + + /** + * Display selected file information. + */ + function displaySelectedFile(file) { + const fileInfo = document.getElementById('selectedFileInfo'); + const fileName = document.getElementById('selectedFileName'); + const fileSize = document.getElementById('selectedFileSize'); + + fileName.textContent = file.name; + fileSize.textContent = `(${formatFileSize(file.size)})`; + fileInfo.style.display = 'block'; + } + + /** + * Upload ZIP file to server. + */ + async function uploadZipFile() { + if (!selectedFile) { + FitPub.showAlert('Please select a ZIP file first', 'warning'); + return; + } + + // Check authentication + const token = localStorage.getItem('jwtToken'); + if (!token) { + FitPub.showAlert('You must be logged in to upload files. Redirecting to login...', 'warning'); + setTimeout(() => { + window.location.href = '/login?redirect=' + encodeURIComponent(window.location.pathname); + }, 2000); + return; + } + + // Validate file size (500 MB max) + const maxSize = 500 * 1024 * 1024; + if (selectedFile.size > maxSize) { + FitPub.showAlert('ZIP file is too large. Maximum size is 500 MB', 'danger'); + return; + } + + try { + // Show progress section + const progressSection = document.getElementById('progressSection'); + progressSection.classList.add('active'); + document.getElementById('statusMessage').textContent = 'Uploading ZIP file...'; + document.getElementById('selectedFileInfo').style.display = 'none'; + + // Create form data + const formData = new FormData(); + formData.append('file', selectedFile); + + // Upload file + // Note: Don't set Content-Type header - browser will set it automatically with boundary + console.log('Uploading ZIP file with token:', token ? 'Present' : 'Missing'); + const response = await fetch('/api/batch-import/upload', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}` + // Don't set Content-Type - browser will set multipart/form-data with boundary + }, + body: formData + }); + + console.log('Upload response status:', response.status); + + if (!response.ok) { + let errorMessage = 'Upload failed'; + try { + const error = await response.json(); + errorMessage = error.error || error.message || `Server returned ${response.status}: ${response.statusText}`; + } catch (e) { + errorMessage = `Server returned ${response.status}: ${response.statusText}`; + } + throw new Error(errorMessage); + } + + const job = await response.json(); + currentJobId = job.id; + + // Start polling for progress + startPolling(job.id); + + // Update UI with initial job info + updateProgress(job); + + FitPub.showAlert(`Batch import started! Processing ${job.totalFiles} files...`, 'success'); + + } catch (error) { + console.error('Upload failed:', error); + FitPub.showAlert('Failed to upload ZIP file: ' + error.message, 'danger'); + document.getElementById('progressSection').classList.remove('active'); + } + } + + /** + * Start polling for job progress. + */ + function startPolling(jobId) { + // Clear any existing interval + if (pollingInterval) { + clearInterval(pollingInterval); + } + + // Poll every 3 seconds + pollingInterval = setInterval(() => { + fetchJobStatus(jobId); + }, 3000); + + // Fetch immediately + fetchJobStatus(jobId); + } + + /** + * Fetch job status from server. + */ + async function fetchJobStatus(jobId) { + try { + const response = await authenticatedFetch(`/api/batch-import/jobs/${jobId}/status`); + + if (!response.ok) { + throw new Error('Failed to fetch job status'); + } + + const job = await response.json(); + updateProgress(job); + + // Stop polling if job is finished + if (job.status === 'COMPLETED' || job.status === 'FAILED' || job.status === 'CANCELLED') { + stopPolling(); + loadFileResults(jobId); + loadRecentJobs(); + + if (job.status === 'COMPLETED') { + FitPub.showAlert(`Batch import completed! ${job.successCount} successful, ${job.failedCount} failed.`, 'success'); + } else if (job.status === 'FAILED') { + FitPub.showAlert('Batch import failed: ' + (job.errorMessage || 'Unknown error'), 'danger'); + } + } + + } catch (error) { + console.error('Failed to fetch job status:', error); + stopPolling(); + } + } + + /** + * Stop polling for progress. + */ + function stopPolling() { + if (pollingInterval) { + clearInterval(pollingInterval); + pollingInterval = null; + } + } + + /** + * Update progress UI with job data. + */ + function updateProgress(job) { + // Update progress bar + const progressBar = document.getElementById('progressBar'); + const progressText = document.getElementById('progressText'); + const percentage = job.progressPercentage || 0; + + progressBar.style.width = percentage + '%'; + progressBar.setAttribute('aria-valuenow', percentage); + progressText.textContent = percentage + '%'; + + // Update stats + document.getElementById('statTotal').textContent = job.totalFiles || 0; + document.getElementById('statProcessed').textContent = job.processedFiles || 0; + document.getElementById('statSuccess').textContent = job.successCount || 0; + document.getElementById('statFailed').textContent = job.failedCount || 0; + + // Update status message + const statusMessage = document.getElementById('statusMessage'); + if (job.status === 'PENDING') { + statusMessage.textContent = 'Starting batch import...'; + } else if (job.status === 'PROCESSING') { + statusMessage.textContent = `Processing files... (${job.processedFiles}/${job.totalFiles})`; + } else if (job.status === 'COMPLETED') { + statusMessage.textContent = 'Batch import completed!'; + progressBar.classList.remove('progress-bar-animated'); + } else if (job.status === 'FAILED') { + statusMessage.textContent = 'Batch import failed!'; + progressBar.classList.remove('progress-bar-animated'); + progressBar.classList.add('bg-danger'); + } + } + + /** + * Load file results when job is complete. + */ + async function loadFileResults(jobId) { + try { + const response = await authenticatedFetch(`/api/batch-import/jobs/${jobId}/files`); + + if (!response.ok) { + throw new Error('Failed to fetch file results'); + } + + const results = await response.json(); + displayFileResults(results); + + } catch (error) { + console.error('Failed to load file results:', error); + } + } + + /** + * Display file results in table. + */ + function displayFileResults(results) { + const section = document.getElementById('fileResultsSection'); + const tbody = document.getElementById('fileResultsBody'); + + tbody.innerHTML = ''; + + results.forEach(result => { + const row = document.createElement('tr'); + + // Filename + const filenameCell = document.createElement('td'); + filenameCell.textContent = result.filename; + row.appendChild(filenameCell); + + // Status + const statusCell = document.createElement('td'); + const statusBadge = document.createElement('span'); + statusBadge.className = `status-badge status-${result.status}`; + statusBadge.textContent = result.status; + statusCell.appendChild(statusBadge); + row.appendChild(statusCell); + + // Activity link + const activityCell = document.createElement('td'); + if (result.activityId) { + const link = document.createElement('a'); + link.href = `/activities/${result.activityId}`; + link.textContent = 'View Activity'; + link.className = 'btn btn-sm btn-outline-primary'; + activityCell.appendChild(link); + } else { + activityCell.textContent = '-'; + } + row.appendChild(activityCell); + + // Error message + const errorCell = document.createElement('td'); + if (result.errorMessage) { + const errorText = document.createElement('small'); + errorText.className = 'text-danger'; + errorText.textContent = result.errorMessage; + errorCell.appendChild(errorText); + } else { + errorCell.textContent = '-'; + } + row.appendChild(errorCell); + + tbody.appendChild(row); + }); + + section.style.display = 'block'; + } + + /** + * Load recent batch import jobs. + */ + async function loadRecentJobs() { + try { + const response = await authenticatedFetch('/api/batch-import/jobs?page=0&size=5'); + + if (!response.ok) { + throw new Error('Failed to fetch recent jobs'); + } + + const page = await response.json(); + displayRecentJobs(page.content || []); + + } catch (error) { + console.error('Failed to load recent jobs:', error); + document.getElementById('recentJobsList').innerHTML = '

Failed to load recent imports

'; + } + } + + /** + * Display recent jobs list. + */ + function displayRecentJobs(jobs) { + const container = document.getElementById('recentJobsList'); + + if (jobs.length === 0) { + container.innerHTML = '

No recent batch imports

'; + return; + } + + container.innerHTML = ''; + + jobs.forEach(job => { + const card = document.createElement('div'); + card.className = 'job-card'; + + const header = document.createElement('div'); + header.className = 'd-flex justify-content-between align-items-center mb-2'; + + const title = document.createElement('h5'); + title.className = 'mb-0'; + title.innerHTML = ` ${job.filename}`; + header.appendChild(title); + + const statusBadge = document.createElement('span'); + statusBadge.className = `status-badge status-${job.status}`; + statusBadge.textContent = job.status; + header.appendChild(statusBadge); + + card.appendChild(header); + + const stats = document.createElement('div'); + stats.className = 'text-muted small'; + stats.innerHTML = ` + ${formatDate(job.createdAt)} | + ${job.totalFiles} files | + ${job.successCount} successful | + ${job.failedCount} failed + `; + card.appendChild(stats); + + // View details button for completed jobs + if (job.status === 'COMPLETED' || job.status === 'FAILED') { + const viewButton = document.createElement('button'); + viewButton.className = 'btn btn-sm btn-outline-primary mt-2'; + viewButton.textContent = 'View Details'; + viewButton.onclick = () => viewJobDetails(job.id); + card.appendChild(viewButton); + } + + container.appendChild(card); + }); + } + + /** + * View job details (load and display file results). + */ + function viewJobDetails(jobId) { + currentJobId = jobId; + fetchJobStatus(jobId).then(() => { + loadFileResults(jobId); + document.getElementById('progressSection').classList.add('active'); + window.scrollTo({ top: 0, behavior: 'smooth' }); + }); + } + + /** + * Format file size for display. + */ + function formatFileSize(bytes) { + if (bytes === 0) return '0 Bytes'; + + const k = 1024; + const sizes = ['Bytes', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i]; + } + + /** + * Format date for display. + */ + function formatDate(dateString) { + const date = new Date(dateString); + const now = new Date(); + const diffMs = now - date; + const diffMins = Math.floor(diffMs / 60000); + + if (diffMins < 1) return 'Just now'; + if (diffMins < 60) return `${diffMins}m ago`; + + const diffHours = Math.floor(diffMins / 60); + if (diffHours < 24) return `${diffHours}h ago`; + + const diffDays = Math.floor(diffHours / 24); + if (diffDays < 7) return `${diffDays}d ago`; + + return date.toLocaleDateString(); + } + + /** + * Authenticated fetch wrapper (uses FitPubAuth from auth.js). + */ + async function authenticatedFetch(url, options = {}) { + if (typeof FitPubAuth !== 'undefined' && FitPubAuth.authenticatedFetch) { + return FitPubAuth.authenticatedFetch(url, options); + } else { + // Fallback if auth.js is not loaded + const token = localStorage.getItem('authToken'); + if (token) { + options.headers = options.headers || {}; + options.headers['Authorization'] = `Bearer ${token}`; + } + return fetch(url, options); + } + } + +})(); diff --git a/src/main/resources/static/js/heatmap.js b/src/main/resources/static/js/heatmap.js index dab0bf1..09c4aa5 100644 --- a/src/main/resources/static/js/heatmap.js +++ b/src/main/resources/static/js/heatmap.js @@ -126,19 +126,22 @@ function renderHeatmap(data) { heatmapMap.removeLayer(heatLayer); } - // Create heat layer + // Create heat layer with red color scheme heatLayer = L.heatLayer(heatData, { - radius: 25, - blur: 15, + radius: 30, // Increased from 25 for better visibility + blur: 20, // Increased from 15 for smoother appearance maxZoom: 17, - max: 1.0, + max: 0.8, // Reduced from 1.0 to make colors more intense + minOpacity: 0.3, // Minimum opacity for better visibility gradient: { - 0.0: 'blue', - 0.4: 'cyan', - 0.6: 'lime', - 0.7: 'yellow', - 0.9: 'orange', - 1.0: 'red' + 0.0: 'rgba(0, 0, 0, 0)', // Transparent for low values + 0.2: 'rgba(139, 0, 0, 0.5)', // Dark red with transparency + 0.4: 'rgba(178, 34, 34, 0.7)', // Firebrick red + 0.6: 'rgb(220, 20, 60)', // Crimson + 0.75: 'rgb(255, 69, 0)', // Red-orange + 0.85: 'rgb(255, 140, 0)', // Dark orange + 0.95: 'rgb(255, 215, 0)', // Gold + 1.0: 'rgb(255, 255, 0)' // Yellow (highest intensity) } }).addTo(heatmapMap); diff --git a/src/main/resources/templates/activities/batch-upload.html b/src/main/resources/templates/activities/batch-upload.html new file mode 100644 index 0000000..73c3e13 --- /dev/null +++ b/src/main/resources/templates/activities/batch-upload.html @@ -0,0 +1,225 @@ + + + + Batch Import - FitPub + + + +
+

Batch Import Activities

+

Upload a ZIP file containing multiple FIT or GPX files for batch processing.

+ + +
+ +

Drop ZIP file here or click to browse

+

Maximum file size: 500 MB | Maximum 1000 files per ZIP

+

Supported formats: .fit, .gpx

+ + +
+ + + + + +
+

Processing...

+
+
+ 0% +
+
+ + +
+
+
0
+
Total Files
+
+
+
0
+
Processed
+
+
+
0
+
Success
+
+
+
0
+
Failed
+
+
+ +

Uploading ZIP file...

+
+ + + + + +
+

Recent Batch Imports

+
+

Loading recent imports...

+
+
+
+ + + + + diff --git a/src/main/resources/templates/heatmap.html b/src/main/resources/templates/heatmap.html index feae716..3bd9b9c 100644 --- a/src/main/resources/templates/heatmap.html +++ b/src/main/resources/templates/heatmap.html @@ -121,10 +121,10 @@ diff --git a/src/main/resources/templates/layout.html b/src/main/resources/templates/layout.html index 69109e7..12fc385 100644 --- a/src/main/resources/templates/layout.html +++ b/src/main/resources/templates/layout.html @@ -70,6 +70,11 @@ Upload +
  • + +