feat: Process fetched timetables and combine stops into services

This commit is contained in:
2025-09-18 17:19:55 +02:00
parent 513acf40c2
commit 050badbd85
19 changed files with 519 additions and 3 deletions
+1
View File
@@ -30,6 +30,7 @@ plugin-jib = "3.4.0"
spring-boot = { module = "org.springframework.boot:spring-boot-dependencies", version.ref = "spring-boot" }
spring-boot-actuator = { module = "org.springframework.boot:spring-boot-starter-actuator" }
spring-boot-web = { module = "org.springframework.boot:spring-boot-starter-web" }
spring-boot-batch = { module = "org.springframework.boot:spring-boot-starter-batch" }
spring-boot-log4j = { module = "org.springframework.boot:spring-boot-starter-log4j2" }
spring-boot-thymeleaf = { module = "org.springframework.boot:spring-boot-starter-thymeleaf" }
spring-boot-mongo = { module = "org.springframework.boot:spring-boot-starter-data-mongodb" }
@@ -4,6 +4,8 @@ import de.twomartens.timetable.bahnApi.model.Eva
import de.twomartens.timetable.bahnApi.model.db.BahnTimetable
import org.bson.types.ObjectId
import org.springframework.data.domain.ExampleMatcher
import org.springframework.data.domain.Page
import org.springframework.data.domain.Pageable
import org.springframework.data.mongodb.repository.MongoRepository
import java.time.ZonedDateTime
@@ -18,4 +20,7 @@ interface BahnTimetableRepository : MongoRepository<BahnTimetable, ObjectId> {
.withMatcher("eva", ExampleMatcher.GenericPropertyMatchers.exact())
.withMatcher("hourAtDay", ExampleMatcher.GenericPropertyMatchers.exact())
}
fun getBahnTimetablesByUserIdAndTswTimetableId(userId: String, tswTimetableId: String,
pageable: Pageable): Page<BahnTimetable>
}
@@ -2,6 +2,8 @@ package de.twomartens.timetable.bahnApi.repository
import de.twomartens.timetable.bahnApi.model.Eva
import de.twomartens.timetable.bahnApi.model.db.ScheduledFetchTask
import de.twomartens.timetable.model.common.TimetableId
import de.twomartens.timetable.model.common.UserId
import de.twomartens.timetable.types.HourAtDay
import org.bson.types.ObjectId
import org.springframework.data.mongodb.repository.MongoRepository
@@ -11,4 +13,6 @@ interface ScheduledFetchTaskRepository : MongoRepository<ScheduledFetchTask, Obj
fun findByEvaAndFetchedDateTime(eva: Eva, fetchedDateTime: HourAtDay): ScheduledFetchTask?
fun findAllByCreatedAfter(createdAt: Instant): List<ScheduledFetchTask>
fun findScheduledFetchTaskByUserIdAndTswTimetableId(userId: UserId, tswTimetableId: TimetableId): List<ScheduledFetchTask>
}
@@ -0,0 +1,35 @@
package de.twomartens.timetable.model.db
import de.twomartens.timetable.model.common.ServiceId
import de.twomartens.timetable.model.common.TimetableId
import de.twomartens.timetable.model.common.UserId
import org.bson.types.ObjectId
import org.springframework.data.annotation.CreatedDate
import org.springframework.data.annotation.Id
import org.springframework.data.annotation.LastModifiedDate
import org.springframework.data.mongodb.core.index.CompoundIndex
import org.springframework.data.mongodb.core.mapping.Document
import java.time.Instant
@Document
@CompoundIndex(def = "{'userId': 1, 'timetableId': 1, 'serviceId': 1}", unique = true)
data class Service(
var userId: UserId,
var timetableId: TimetableId,
var serviceId: ServiceId,
private var stops: MutableList<ServiceStop> = mutableListOf()
) {
@Id
lateinit var id: ObjectId
@CreatedDate
lateinit var created: Instant
@LastModifiedDate
lateinit var lastModified: Instant
fun addStop(stop: ServiceStop) {
stops.add(stop)
stops.sortWith(compareBy { it.plannedDepartureTime ?: it.plannedArrivalTime })
}
}
@@ -0,0 +1,14 @@
package de.twomartens.timetable.model.db
import de.twomartens.timetable.model.common.ServiceId
import de.twomartens.timetable.model.common.TimetableId
import de.twomartens.timetable.model.common.UserId
import org.bson.types.ObjectId
import org.springframework.data.mongodb.repository.MongoRepository
import org.springframework.stereotype.Repository
@Repository
interface ServiceRepository : MongoRepository<Service, ObjectId> {
fun existsServiceByUserIdAndTimetableIdAndServiceId(userId: UserId, timetableId: TimetableId, serviceId: ServiceId): Boolean
fun getServiceByUserIdAndTimetableIdAndServiceId(userId: UserId, timetableId: TimetableId, serviceId: ServiceId): Service?
}
@@ -0,0 +1,32 @@
package de.twomartens.timetable.model.db
import java.time.ZonedDateTime
data class ServiceStop(
/** The EVA number of the station */
val eva: String,
/** Planned arrival time at this stop */
var plannedArrivalTime: ZonedDateTime? = null,
/** Planned departure time from this stop */
var plannedDepartureTime: ZonedDateTime? = null,
/** Platform for arrival */
var plannedArrivalPlatform: String? = null,
/** Platform for departure */
var plannedDeparturePlatform: String? = null,
/** Line information for arrival */
var arrivalLine: String? = null,
/** Line information for departure */
var departureLine: String? = null,
/** Path information for arrival */
var plannedArrivalPath: String? = null,
/** Path information for departure */
var plannedDeparturePath: String? = null
)
@@ -2,7 +2,9 @@ package de.twomartens.timetable.model.dto
enum class TimetableState {
NEW,
PROCESSING,
FETCHING_TIMETABLES,
TIMETABLES_FETCHED,
SERVICES_COLLECTED,
ENTER_FORMATIONS,
LINK_SERVICES,
READY_FOR_USAGE
+1
View File
@@ -17,5 +17,6 @@ dependencies {
annotationProcessor(libs.mapstruct.processor)
kapt(libs.mapstruct.processor)
implementation(libs.spring.boot.batch)
implementation(libs.bundles.spring.boot.security)
}
@@ -0,0 +1,195 @@
package de.twomartens.timetable.service
import de.twomartens.timetable.bahnApi.model.db.BahnTimetable
import de.twomartens.timetable.bahnApi.repository.BahnTimetableRepository
import de.twomartens.timetable.model.common.ServiceId
import de.twomartens.timetable.model.common.TimetableId
import de.twomartens.timetable.model.common.UserId
import de.twomartens.timetable.model.db.Service
import de.twomartens.timetable.model.db.ServiceRepository
import de.twomartens.timetable.model.db.ServiceStop
import de.twomartens.timetable.model.db.Timetable
import de.twomartens.timetable.model.dto.TimetableState
import de.twomartens.timetable.timetable.TimetableRepository
import de.twomartens.timetable.types.NonEmptyString
import org.springframework.batch.core.Job
import org.springframework.batch.core.JobParameters
import org.springframework.batch.core.Step
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing
import org.springframework.batch.core.configuration.annotation.StepScope
import org.springframework.batch.core.job.builder.JobBuilder
import org.springframework.batch.core.launch.JobLauncher
import org.springframework.batch.core.launch.support.TaskExecutorJobLauncher
import org.springframework.batch.core.repository.JobRepository
import org.springframework.batch.core.step.builder.StepBuilder
import org.springframework.batch.item.ItemProcessor
import org.springframework.batch.item.data.builder.RepositoryItemReaderBuilder
import org.springframework.batch.item.data.builder.RepositoryItemWriterBuilder
import org.springframework.beans.factory.annotation.Value
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.core.task.SimpleAsyncTaskExecutor
import org.springframework.data.domain.Sort
import org.springframework.transaction.PlatformTransactionManager
@EnableBatchProcessing
@Configuration
class BatchProcessTimetableConfiguration {
@Bean("asyncJobLauncher")
fun jobLauncher(jobRepository: JobRepository): JobLauncher {
val launcher = TaskExecutorJobLauncher()
launcher.setJobRepository(jobRepository)
launcher.setTaskExecutor(SimpleAsyncTaskExecutor())
launcher.afterPropertiesSet()
return launcher
}
@Bean
fun processTimetableJob(jobRepository: JobRepository,
createStopStorageRequestsStep: Step,
storeStopsStep: Step): Job {
val job = JobBuilder("processTimetable", jobRepository)
.start(createStopStorageRequestsStep)
.next(storeStopsStep)
.build()
return job
}
@Bean("createStopStorageRequestsStep")
@StepScope
fun createStopStorageRequestsStep(
jobRepository: JobRepository,
transactionManager: PlatformTransactionManager,
bahnTimetableRepository: BahnTimetableRepository,
stopStorageRequestRepository: StopStorageRequestRepository,
@Value("#{jobParameters}") jobParameters: JobParameters): Step {
val reader = RepositoryItemReaderBuilder<BahnTimetable>()
.repository(bahnTimetableRepository)
.methodName("getBahnTimetablesByUserIdAndTswTimetableId")
.name("BahnTimetableReader")
.arguments(
UserId.of(NonEmptyString(jobParameters.getString("userId")!!)),
TimetableId.of(NonEmptyString(jobParameters.getString("timetableId")!!))
)
.sorts(mapOf("eva" to Sort.Direction.ASC, "hourAtDay" to Sort.Direction.ASC))
.pageSize(26) // one station per page
.saveState(true)
.build()
val processor = ItemProcessor<BahnTimetable, List<StopStorageRequest>> { timetable ->
timetable.stops.map {
StopStorageRequest(
userId = timetable.userId.toString(),
timetableId = timetable.tswTimetableId.value,
eva = it.eva.toString(),
trainNumber = it.tripLabel.trainNumber,
plannedArrivalTime = it.arrival?.plannedTime,
plannedDepartureTime = it.departure?.plannedTime,
plannedArrivalPlatform = it.arrival?.plannedPlatform,
plannedDeparturePlatform = it.departure?.plannedPlatform,
arrivalLine = it.arrival?.line,
departureLine = it.departure?.line,
plannedArrivalPath = it.arrival?.plannedPath,
plannedDeparturePath = it.departure?.plannedPath
)
}
}
return StepBuilder("createStopStorageRequests", jobRepository)
.chunk<BahnTimetable, List<StopStorageRequest>>(10, transactionManager)
.reader(reader)
.processor(processor)
.writer {
val items = it.items.flatMap { requests -> requests }
stopStorageRequestRepository.saveAll(items)
}
.build()
}
@Bean("storeStopsStep")
@StepScope
fun storeStopsStep(jobRepository: JobRepository,
transactionManager: PlatformTransactionManager,
stopStorageRequestRepository: StopStorageRequestRepository,
serviceRepository: ServiceRepository,
@Value("#{jobParameters}") jobParameters: JobParameters): Step {
val userId = UserId.of(NonEmptyString(jobParameters.getString("userId")!!))
val timetableId = TimetableId.of(NonEmptyString(jobParameters.getString("timetableId")!!))
val reader = RepositoryItemReaderBuilder<StopStorageRequest>()
.repository(stopStorageRequestRepository)
.methodName("getStopStorageRequestsByUserIdAndTimetableId")
.name("StopStorageRequestReader")
.arguments(userId, timetableId)
.sorts(mapOf("eva" to Sort.Direction.ASC))
.pageSize(10)
.saveState(true)
.build()
val processor = ItemProcessor<StopStorageRequest, Service> {
val serviceId = ServiceId.of(NonEmptyString(it.trainNumber))
val serviceExists = serviceRepository.existsServiceByUserIdAndTimetableIdAndServiceId(
userId, timetableId, serviceId)
val service: Service = if (serviceExists) {
serviceRepository.getServiceByUserIdAndTimetableIdAndServiceId(
userId, timetableId, serviceId)!!
} else {
Service(userId, timetableId, serviceId)
}
service.addStop(ServiceStop(
it.eva,
it.plannedArrivalTime,
it.plannedDepartureTime,
it.plannedArrivalPlatform,
it.plannedDeparturePlatform,
it.arrivalLine,
it.departureLine,
it.plannedArrivalPath,
it.plannedDeparturePath
))
service
}
val writer = RepositoryItemWriterBuilder<Service>()
.repository(serviceRepository)
.methodName("save")
.build()
return StepBuilder("storeStopsStep", jobRepository)
.chunk<StopStorageRequest, Service>(26, transactionManager)
.reader(reader)
.processor(processor)
.writer(writer)
.build()
}
@Bean("updateTimetableStateStep")
@StepScope
fun updateTimetableStateStep(jobRepository: JobRepository,
transactionManager: PlatformTransactionManager,
timetableRepository: TimetableRepository,
@Value("#{jobParameters}") jobParameters: JobParameters): Step {
val reader = RepositoryItemReaderBuilder<Timetable>()
.repository(timetableRepository)
.methodName("getTimetableByUserIdAndTimetableId")
.name("TimetableReader")
.arguments(
UserId.of(NonEmptyString(jobParameters.getString("userId")!!)),
TimetableId.of(NonEmptyString(jobParameters.getString("timetableId")!!))
)
.pageSize(1)
.build()
val processor = ItemProcessor<Timetable, Timetable> {
it.timetableState = TimetableState.SERVICES_COLLECTED
it
}
val writer = RepositoryItemWriterBuilder<Timetable>()
.repository(timetableRepository)
.methodName("save")
.build()
return StepBuilder("updateTimetableStateStep", jobRepository)
.chunk<Timetable, Timetable>(1, transactionManager)
.reader(reader)
.processor(processor)
.writer(writer)
.build()
}
}
@@ -0,0 +1,49 @@
package de.twomartens.timetable.service
import de.twomartens.timetable.model.db.Timetable
import de.twomartens.timetable.model.dto.TimetableState
import mu.KotlinLogging
import org.springframework.batch.core.Job
import org.springframework.batch.core.JobParametersBuilder
import org.springframework.batch.core.launch.JobInstanceAlreadyExistsException
import org.springframework.batch.core.launch.JobLauncher
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException
import org.springframework.scheduling.annotation.Async
@Async
class ProcessTimetableTask(
private val asyncJobLauncher: JobLauncher,
private val processTimetableJob: Job,
private val tswTimetable: Timetable,
private val repository: ScheduledProcessTimetableTaskRepository,
private val scheduledTask: ScheduledProcessTimetableTask) : Runnable {
override fun run() {
processTimetable()
repository.delete(scheduledTask)
}
fun processTimetable() {
if (tswTimetable.timetableState != TimetableState.TIMETABLES_FETCHED) {
log.info("Timetable ${tswTimetable.timetableId} is not in TIMETABLES_FETCHED state")
return
}
val jobParameters = JobParametersBuilder()
.addString("timetableId", tswTimetable.timetableId.value)
.addString("userId", tswTimetable.userId.value)
.toJobParameters()
try {
asyncJobLauncher.run(processTimetableJob, jobParameters)
} catch (_: JobExecutionAlreadyRunningException) {
log.info("Job is already running")
} catch (_: JobInstanceAlreadyExistsException) {
log.info("Job instance already exists")
}
}
companion object {
private val log = KotlinLogging.logger {}
}
}
@@ -0,0 +1,6 @@
package de.twomartens.timetable.service
import org.springframework.context.ApplicationEvent
class ProcessTimetableTaskScheduledEvent(source: ScheduledProcessTimetableTask) : ApplicationEvent(source) {
}
@@ -0,0 +1,30 @@
package de.twomartens.timetable.service
import de.twomartens.timetable.model.common.TimetableId
import de.twomartens.timetable.model.common.UserId
import de.twomartens.timetable.model.db.Timetable
import org.bson.types.ObjectId
import org.springframework.data.annotation.CreatedDate
import org.springframework.data.annotation.Id
import org.springframework.data.annotation.LastModifiedDate
import org.springframework.data.mongodb.core.index.CompoundIndex
import org.springframework.data.mongodb.core.mapping.Document
import java.time.Instant
@Document
@CompoundIndex(def = "{'userId': 1, 'tswTimetableId': 1}")
data class ScheduledProcessTimetableTask(
var userId: UserId,
var timetableId: TimetableId,
var timetable: Timetable,
var executionTime: Instant
) {
@Id
lateinit var id: ObjectId
@CreatedDate
lateinit var created: Instant
@LastModifiedDate
lateinit var lastModified: Instant
}
@@ -0,0 +1,9 @@
package de.twomartens.timetable.service
import org.bson.types.ObjectId
import org.springframework.data.mongodb.repository.MongoRepository
import java.time.Instant
interface ScheduledProcessTimetableTaskRepository : MongoRepository<ScheduledProcessTimetableTask, ObjectId> {
fun findScheduledProcessTimetableTasksByExecutionTimeAfter(executionTime: Instant): List<ScheduledProcessTimetableTask>
}
@@ -0,0 +1,33 @@
package de.twomartens.timetable.service
import org.bson.types.ObjectId
import org.springframework.data.annotation.CreatedDate
import org.springframework.data.annotation.Id
import org.springframework.data.annotation.LastModifiedDate
import org.springframework.data.mongodb.core.mapping.Document
import java.time.Instant
import java.time.ZonedDateTime
@Document
data class StopStorageRequest(
var userId: String,
var timetableId: String,
var eva: String,
var trainNumber: String,
var plannedArrivalTime: ZonedDateTime?,
var plannedDepartureTime: ZonedDateTime?,
var plannedArrivalPlatform: String?,
var plannedDeparturePlatform: String?,
var arrivalLine: String?,
var departureLine: String?,
var plannedArrivalPath: String?,
var plannedDeparturePath: String?) {
@Id
lateinit var id: ObjectId
@CreatedDate
lateinit var created: Instant
@LastModifiedDate
lateinit var lastModified: Instant
}
@@ -0,0 +1,13 @@
package de.twomartens.timetable.service
import org.bson.types.ObjectId
import org.springframework.data.domain.Page
import org.springframework.data.domain.Pageable
import org.springframework.data.mongodb.repository.MongoRepository
interface StopStorageRequestRepository : MongoRepository<StopStorageRequest, ObjectId> {
// is used
fun getStopStorageRequestsByUserIdAndTimetableId(userId: String, timetableId: String, pageable: Pageable): Page<StopStorageRequest>
fun saveAll(stopStorageRequests: List<StopStorageRequest>): List<StopStorageRequest>
}
@@ -0,0 +1,49 @@
package de.twomartens.timetable.service
import de.twomartens.timetable.model.db.Timetable
import org.springframework.batch.core.Job
import org.springframework.batch.core.launch.JobLauncher
import org.springframework.boot.context.event.ApplicationReadyEvent
import org.springframework.context.ApplicationEventPublisher
import org.springframework.context.event.EventListener
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler
import org.springframework.stereotype.Service
import java.time.Clock
@Service
class TimetableProcessingScheduler(
private val processTimetableJob: Job,
private val asyncJobLauncher: JobLauncher,
private val threadPoolTaskScheduler: ThreadPoolTaskScheduler,
private val repository: ScheduledProcessTimetableTaskRepository,
private val clock: Clock,
private val eventPublisher: ApplicationEventPublisher
) {
@EventListener(ApplicationReadyEvent::class)
fun onApplicationReady(event: ApplicationReadyEvent) {
val tasks = repository.findScheduledProcessTimetableTasksByExecutionTimeAfter(clock.instant())
tasks.forEach {
val task = ProcessTimetableTask(asyncJobLauncher, processTimetableJob,
it.timetable, repository, it)
threadPoolTaskScheduler.schedule(task, it.executionTime)
}
}
@EventListener(ProcessTimetableTaskScheduledEvent::class)
fun onProcessTimetableTaskScheduled(event: ProcessTimetableTaskScheduledEvent) {
val source = event.source as ScheduledProcessTimetableTask
val task = ProcessTimetableTask(asyncJobLauncher, processTimetableJob,
source.timetable, repository, source)
threadPoolTaskScheduler.schedule(task, source.executionTime)
}
fun scheduleTimetableProcessing(timetable: Timetable) {
val task = ScheduledProcessTimetableTask(timetable.userId, timetable.timetableId,
timetable, timetable.fetchDate.plusDays(2)
.atStartOfDay(clock.zone).toInstant())
repository.save(task)
val event = ProcessTimetableTaskScheduledEvent(task)
eventPublisher.publishEvent(event)
}
}
@@ -7,6 +7,7 @@ import de.twomartens.timetable.model.common.UserId
import de.twomartens.timetable.model.dto.Timetable
import de.twomartens.timetable.model.dto.TimetableState
import de.twomartens.timetable.route.TswRouteRepository
import de.twomartens.timetable.service.TimetableProcessingScheduler
import de.twomartens.timetable.types.NonEmptyString
import io.swagger.v3.oas.annotations.Operation
import io.swagger.v3.oas.annotations.Parameter
@@ -32,7 +33,8 @@ class TimetableController(
private val routeRepository: TswRouteRepository,
private val timetableRepository: TimetableRepository,
private val userRepository: UserRepository,
private val scheduledTaskService: ScheduledTaskService
private val scheduledTaskService: ScheduledTaskService,
private val timetableProcessingScheduler: TimetableProcessingScheduler
) {
private val mapper = Mappers.getMapper(TimetableMapper::class.java)
@@ -171,7 +173,7 @@ class TimetableController(
if (timetable == null) {
created = true
timetable = mapper.mapToDB(userIdConverted, body)
timetable.timetableState = TimetableState.PROCESSING
timetable.timetableState = TimetableState.FETCHING_TIMETABLES
if (!timetable.fetchDate.isAfter(LocalDate.now(clock))) {
throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Date of timetable must be in the future")
@@ -183,6 +185,7 @@ class TimetableController(
)
scheduledTaskService.triggerTimetableFetch(route, timetable.timetableId,
timetable.fetchDate)
timetableProcessingScheduler.scheduleTimetableProcessing(timetable)
} else {
timetable.name = NonEmptyString(body.name)
}
@@ -4,10 +4,15 @@ import de.twomartens.timetable.model.common.TimetableId
import de.twomartens.timetable.model.common.UserId
import de.twomartens.timetable.model.db.Timetable
import org.bson.types.ObjectId
import org.springframework.data.domain.Page
import org.springframework.data.mongodb.repository.MongoRepository
import java.awt.print.Pageable
interface TimetableRepository : MongoRepository<Timetable, ObjectId> {
fun findByUserIdAndTimetableId(userId: UserId, timetableId: TimetableId): Timetable?
fun findAllByUserIdAndNameContainingIgnoreCase(userId: UserId, name: String): List<Timetable>
fun findAllByUserId(userId: UserId): List<Timetable>
// is used
fun getTimetableByUserIdAndTimetableId(userId: UserId, timetableId: TimetableId, pageable: Pageable): Page<Timetable>
}
@@ -0,0 +1,30 @@
package de.twomartens.timetable.timetable
import de.twomartens.timetable.bahnApi.events.ScheduledTaskDeletedEvent
import de.twomartens.timetable.bahnApi.repository.ScheduledFetchTaskRepository
import de.twomartens.timetable.bahnApi.tasks.DeleteScheduledTask
import de.twomartens.timetable.model.dto.TimetableState
import org.springframework.context.event.EventListener
import org.springframework.stereotype.Service
@Service
class TimetableStateUpdater(
private val fetchTaskRepository: ScheduledFetchTaskRepository,
private val timetableRepository: TimetableRepository
) {
@EventListener(ScheduledTaskDeletedEvent::class)
fun onScheduledTaskDeleted(event: ScheduledTaskDeletedEvent) {
val source = event.source as DeleteScheduledTask
val tasks = fetchTaskRepository.findScheduledFetchTaskByUserIdAndTswTimetableId(source.scheduledFetchTask.userId,
source.scheduledFetchTask.tswTimetableId)
if (tasks.isEmpty()) {
val timetable = timetableRepository.findByUserIdAndTimetableId(source.scheduledFetchTask.userId,
source.scheduledFetchTask.tswTimetableId)
timetable?.timetableState = TimetableState.TIMETABLES_FETCHED
if (timetable != null) {
timetableRepository.save(timetable)
}
}
}
}