From df7c68788926f1f322e51ca83ab9a28386e15ea9 Mon Sep 17 00:00:00 2001 From: Jinil Sung Date: Tue, 19 Dec 2023 08:42:53 -0800 Subject: [PATCH 1/5] GRAD2-2430: task is complete. GRAD2-2430: task is complete. --- .../config/BatchJobLauncher.java | 13 +++++++ ...atchGradAlgorithmJobHistoryRepository.java | 8 +++++ .../BatchGradAlgorithmStudentRepository.java | 9 ++++- .../BatchJobExecutionRepository.java | 31 ++++++++++++++++ .../BatchStepExecutionRepository.java | 20 +++++++++++ ...udentCredentialDistributionRepository.java | 10 ++++-- .../service/GradDashboardService.java | 36 ++++++++++++++++++- .../EducGradBatchGraduationApiConstants.java | 3 ++ api/src/main/resources/application.yaml | 3 ++ .../service/GradDashboardServiceTest.java | 22 ++++++++++++ api/src/test/resources/application.yaml | 3 ++ 11 files changed, 153 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java index c2789099..8c7ec12f 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java @@ -146,4 +146,17 @@ public void refreshUserScheduledQueue() { LOGGER.info(BATCH_ENDED); } + + @Scheduled(cron = "${batch.purge-old-records.cron}") + @SchedulerLock(name = "PurgeOldRecordsLock", + lockAtLeastFor = "PT1H", lockAtMostFor = "PT1H") //midnight job so lock for an hour + public void purgeOldRecords() { + LockAssert.assertLocked(); + try { + this.gradDashboardService.purgeOldBatchHistoryRecords(); + this.gradDashboardService.purgeOldSpringMetaDataRecords(); + } catch (Exception e) { + LOGGER.error(ERROR_MSG, e.getLocalizedMessage()); + } + } } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java index 6a8588b7..dbab9180 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java @@ -1,13 +1,17 @@ package ca.bc.gov.educ.api.batchgraduation.repository; +import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity; +import org.springframework.transaction.annotation.Transactional; @Repository public interface BatchGradAlgorithmJobHistoryRepository extends JpaRepository { @@ -16,4 +20,8 @@ public interface BatchGradAlgorithmJobHistoryRepository extends JpaRepository findByJobExecutionId(Long batchId); + @Transactional + @Modifying + @Query("delete from BatchGradAlgorithmJobHistoryEntity where createDate <= :createDate") + void deleteByCreateDateBefore(LocalDateTime createDate); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java index 17603024..f2e9ee3e 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java @@ -8,7 +8,9 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.time.LocalDateTime; import java.util.*; @Repository @@ -58,5 +60,10 @@ void copyAllGradAlgorithmStudents( "and status = 'COMPLETED'\n" + "group by graduation_program_code\n" + "order by graduation_program_code desc", nativeQuery = true) - List getGraduationProgramCounts( @Param("batchId") Long batchId); + List getGraduationProgramCounts(@Param("batchId") Long batchId); + + @Transactional + @Modifying + @Query("delete from BatchGradAlgorithmStudentEntity where createDate <= :createDate") + void deleteByCreateDateBefore(LocalDateTime createDate); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java index 81a3f1f5..6f92679e 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java @@ -4,11 +4,42 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; @Repository public interface BatchJobExecutionRepository extends JpaRepository { Page findAllByOrderByCreateTimeDesc(Pageable page); + @Transactional + @Modifying + @Query(value = "DELETE FROM BATCH_JOB_EXECUTION_PARAMS WHERE JOB_EXECUTION_ID IN (\n" + + "SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate);", + nativeQuery = true) + void deleteBatchParamsByCreateTimeBefore(LocalDateTime createDate); + + @Transactional + @Modifying + @Query(value = "DELETE FROM BATCH_JOB_EXECUTION_CONTEXT WHERE JOB_EXECUTION_ID IN (\n" + + "SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate);", + nativeQuery = true) + void deleteBatchContextsByCreateTimeBefore(LocalDateTime createDate); + + @Transactional + @Modifying + @Query(value = "DELETE FROM BATCH_JOB_INSTANCE WHERE JOB_INSTANCE_ID NOT IN (\n" + + "SELECT JOB_INSTANCE_ID FROM BATCH_JOB_EXECUTION);", + nativeQuery = true) + void deleteBatchInstancesNotInBatchJobs(); + + @Transactional + @Modifying + @Query(value = "DELETE FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate;", + nativeQuery = true) + void deleteBatchJobsByCreateTimeBefore(LocalDateTime createDate); } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java index 0d4adcfa..2194ce72 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java @@ -2,8 +2,12 @@ import ca.bc.gov.educ.api.batchgraduation.entity.BatchStepExecutionEntity; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.time.LocalDateTime; import java.util.List; @Repository @@ -11,4 +15,20 @@ public interface BatchStepExecutionRepository extends JpaRepository findByJobExecutionIdOrderByEndTimeDesc(Long jobExecutionId); + @Transactional + @Modifying + @Query(value = "DELETE FROM BATCH_STEP_EXECUTION_CONTEXT WHERE STEP_EXECUTION_ID IN (\n" + + "SELECT BATCH_STEP_EXECUTION.STEP_EXECUTION_ID FROM BATCH_STEP_EXECUTION WHERE JOB_EXECUTION_ID IN (\n" + + " SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate\n" + + "));\n", + nativeQuery = true) + void deleteBatchStepContextsByCreateTimeBefore(LocalDateTime createDate); + + @Transactional + @Modifying + @Query(value = "DELETE FROM BATCH_STEP_EXECUTION WHERE JOB_EXECUTION_ID IN (\n" + + " SELECT JOB_EXECUTION_ID FROM BATCH_JOB_EXECUTION WHERE CREATE_TIME <= :createDate);", + nativeQuery = true) + void deleteBatchStepsByCreateTimeBefore(LocalDateTime createDate); + } diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java index 3b5a152d..6ab5fc13 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java @@ -1,16 +1,15 @@ package ca.bc.gov.educ.api.batchgraduation.repository; -import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmStudentEntity; import ca.bc.gov.educ.api.batchgraduation.entity.StudentCredentialDistributionEntity; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; -import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; +import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -37,4 +36,9 @@ public interface StudentCredentialDistributionRepository extends JpaRepository Date: Tue, 19 Dec 2023 12:18:04 -0800 Subject: [PATCH 2/5] GRAD2-2447: flyway to drop unused table. GRAD2-2447: flyway to drop unused table. --- .../V1.0.44__DDL-DROP_TABLE-batch_job_execution_params_old.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 api/src/main/resources/db/migration/1.0/V1.0.44__DDL-DROP_TABLE-batch_job_execution_params_old.sql diff --git a/api/src/main/resources/db/migration/1.0/V1.0.44__DDL-DROP_TABLE-batch_job_execution_params_old.sql b/api/src/main/resources/db/migration/1.0/V1.0.44__DDL-DROP_TABLE-batch_job_execution_params_old.sql new file mode 100644 index 00000000..fdf870fe --- /dev/null +++ b/api/src/main/resources/db/migration/1.0/V1.0.44__DDL-DROP_TABLE-batch_job_execution_params_old.sql @@ -0,0 +1 @@ +DROP TABLE API_GRAD_BATCH.BATCH_JOB_EXECUTION_PARAMS_OLD; \ No newline at end of file From 3cd5dd029e637f7be7a513d694979101c0501e4f Mon Sep 17 00:00:00 2001 From: Jinil Sung Date: Wed, 20 Dec 2023 11:05:24 -0800 Subject: [PATCH 3/5] GRAD2-2430: bug fix on LocalDateTime compatible with native query. GRAD2-2430: bug fix on LocalDateTime compatible with native query. --- .../entity/BatchJobExecutionContextEntity.java | 2 +- .../BatchStepExecutionContextEntity.java | 2 +- .../entity/BatchStepExecutionEntity.java | 2 +- .../BatchJobExecutionRepository.java | 18 +++++++----------- .../BatchStepExecutionRepository.java | 13 +++++-------- .../service/GradDashboardService.java | 1 + .../service/GradDashboardServiceTest.java | 2 +- 7 files changed, 17 insertions(+), 23 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchJobExecutionContextEntity.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchJobExecutionContextEntity.java index 36da6f86..dcaff691 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchJobExecutionContextEntity.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchJobExecutionContextEntity.java @@ -11,7 +11,7 @@ public class BatchJobExecutionContextEntity { @Id @Column(name = "JOB_EXECUTION_ID", nullable = false) - private Long id; + private Long jobExecutionId; @Column(name = "SHORT_CONTEXT", nullable = false, length = 2500) private String shortContext; diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionContextEntity.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionContextEntity.java index 0d0e684f..6668826c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionContextEntity.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionContextEntity.java @@ -10,7 +10,7 @@ public class BatchStepExecutionContextEntity { @Id @Column(name = "STEP_EXECUTION_ID", nullable = false) - private Long id; + private Long stepExecutionId; @Column(name = "SHORT_CONTEXT", length = 2500) private String shortContext; diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionEntity.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionEntity.java index 6b01c9e8..3f670bac 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionEntity.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionEntity.java @@ -12,7 +12,7 @@ public class BatchStepExecutionEntity { @Id @Column(name = "STEP_EXECUTION_ID", nullable = false) - private Long id; + private Long stepExecutionId; @Column(name = "JOB_EXECUTION_ID", nullable = false) private Long jobExecutionId; diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java index 6f92679e..5f8d278c 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java @@ -18,28 +18,24 @@ public interface BatchJobExecutionRepository extends JpaRepository Date: Wed, 20 Dec 2023 11:11:39 -0800 Subject: [PATCH 4/5] Cleanup unused imports. Cleanup unused imports. --- .../educ/api/batchgraduation/service/GradDashboardService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java index eebd95f5..3aaacb28 100644 --- a/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java +++ b/api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java @@ -6,7 +6,6 @@ import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils; import ca.bc.gov.educ.api.batchgraduation.transformer.BatchGradAlgorithmJobHistoryTransformer; import ca.bc.gov.educ.api.batchgraduation.transformer.BatchProcessingTransformer; -import ca.bc.gov.educ.api.batchgraduation.util.DateUtils; import ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; From ec2683cfd7f729cac535098de6621a5f6652ab65 Mon Sep 17 00:00:00 2001 From: Kamal Mohammed Date: Mon, 15 Jan 2024 10:12:44 -0700 Subject: [PATCH 5/5] Update pom.xml --- api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/pom.xml b/api/pom.xml index 7772c9ab..74e6d433 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -6,7 +6,7 @@ ca.bc.gov.educ educ-grad-batch-graduation-api - 1.8.54 + 1.8.55 educ-grad-batch-graduation-api Ministry of Education GRAD BATCH GRADUATION API