Skip to content

Commit

Permalink
Merge pull request #516 from bcgov/develop/alex-GRAD2-2817
Browse files Browse the repository at this point in the history 
Develop/alex grad2 2817
  • Loading branch information
@arybakov-cgi
arybakov-cgi authored Aug 28, 2024
2 parents d50af14 + 0e6e3be commit f7ffa4e
Show file tree
Hide file tree
Showing 17 changed files with 220 additions and 108 deletions.
50 changes: 41 additions & 9 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/controller/JobLauncherController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,31 @@ private void validateInputDeleteStudentReports(StudentSearchRequest studentSearc
if(studentSearchRequest == null) {
throw new GradBusinessRuleException("Please provide not null student search request");
}
StringBuilder errors = new StringBuilder();
if(studentSearchRequest.isEmpty() && StringUtils.isBlank(studentSearchRequest.getActivityCode())) {
errors.append("Please provide school of records or set activityCode to ALL to delete all reports").append('\n');
}
if(studentSearchRequest.getReportTypes().isEmpty()) {
errors.append("Please provide at least 1 report type code").append('\n');
}
String errorsAsString = errors.toString();
if(StringUtils.isNotBlank(errorsAsString)) {
throw new GradBusinessRuleException(errorsAsString);
}
}

private void validateInputArchiveStudents(StudentSearchRequest studentSearchRequest) {
if(studentSearchRequest == null) {
throw new GradBusinessRuleException("Please provide not null student search request");
}
StringBuilder errors = new StringBuilder();
if(studentSearchRequest.isEmpty() && StringUtils.isBlank(studentSearchRequest.getActivityCode())) {
errors.append("Please provide school of records or set activityCode to ALL to archive all students").append('\n');
}
String errorsAsString = errors.toString();
if(StringUtils.isNotBlank(errorsAsString)) {
throw new GradBusinessRuleException(errorsAsString);
}
}

private BlankDistributionSummaryDTO validateInputBlankDisRun(BlankCredentialRequest blankCredentialRequest) {
Expand Down Expand Up @@ -877,9 +902,9 @@ public ResponseEntity<BatchJobResponse> launchArchiveSchoolReportsJob(@RequestBo
logger.debug("launchArchiveSchoolReporsJob");
BatchJobResponse response = new BatchJobResponse();
JobParametersBuilder builder = new JobParametersBuilder();

String userName = ThreadLocalStateUtil.getCurrentUser();
builder.addLong(TIME, System.currentTimeMillis()).toJobParameters();
builder.addString(RUN_BY, ThreadLocalStateUtil.getCurrentUser());
builder.addString(RUN_BY, userName);
builder.addString(JOB_TRIGGER, MANUAL);
builder.addString(JOB_TYPE, ARCHIVE_SCHOOL_REPORTS);

Expand All @@ -899,12 +924,14 @@ public ResponseEntity<BatchJobResponse> launchArchiveSchoolReportsJob(@RequestBo
ExecutionContext jobContext = jobExecution.getExecutionContext();
DistributionSummaryDTO summaryDTO = new DistributionSummaryDTO();
summaryDTO.setBatchId(jobExecution.getId());
summaryDTO.setUserName(userName);
summaryDTO.setStudentSearchRequest(studentSearchRequest);
jobContext.put(DISDTO, summaryDTO);
response.setBatchId(jobExecution.getId());
} else {
response.setBatchId(jobParameters.getLong("run.id"));
}
return ResponseEntity.ok(response);
return ResponseEntity.status(HttpStatus.CREATED).body(response);
} catch (JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException
| JobParametersInvalidException | NoSuchJobException | IllegalArgumentException e) {
response.setException(e.getLocalizedMessage());
Expand All @@ -920,9 +947,9 @@ public ResponseEntity<BatchJobResponse> launchDeleteStudentReportsJob(@RequestBo
logger.debug("launchDeleteStudentReportsJob");
BatchJobResponse response = new BatchJobResponse();
JobParametersBuilder builder = new JobParametersBuilder();

String userName = ThreadLocalStateUtil.getCurrentUser();
builder.addLong(TIME, System.currentTimeMillis()).toJobParameters();
builder.addString(RUN_BY, ThreadLocalStateUtil.getCurrentUser());
builder.addString(RUN_BY, userName);
builder.addString(JOB_TRIGGER, MANUAL);
builder.addString(JOB_TYPE, DELETE_STUDENT_REPORTS);

Expand All @@ -942,12 +969,14 @@ public ResponseEntity<BatchJobResponse> launchDeleteStudentReportsJob(@RequestBo
ExecutionContext jobContext = jobExecution.getExecutionContext();
DistributionSummaryDTO summaryDTO = new DistributionSummaryDTO();
summaryDTO.setBatchId(jobExecution.getId());
summaryDTO.setUserName(userName);
summaryDTO.setStudentSearchRequest(studentSearchRequest);
jobContext.put(DISDTO, summaryDTO);
response.setBatchId(jobExecution.getId());
} else {
response.setBatchId(jobParameters.getLong("run.id"));
}
return ResponseEntity.ok(response);
return ResponseEntity.status(HttpStatus.CREATED).body(response);
} catch (JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException
| JobParametersInvalidException | NoSuchJobException | IllegalArgumentException e) {
response.setException(e.getLocalizedMessage());
Expand All @@ -963,9 +992,9 @@ public ResponseEntity<BatchJobResponse> launchArchiveStudentsJob(@RequestBody St
logger.debug("launchArchiveStudentsJob");
BatchJobResponse response = new BatchJobResponse();
JobParametersBuilder builder = new JobParametersBuilder();

String userName = ThreadLocalStateUtil.getCurrentUser();
builder.addLong(TIME, System.currentTimeMillis()).toJobParameters();
builder.addString(RUN_BY, ThreadLocalStateUtil.getCurrentUser());
builder.addString(RUN_BY, userName);
builder.addString(JOB_TRIGGER, MANUAL);
builder.addString(JOB_TYPE, ARCHIVE_STUDENTS);

Expand All @@ -975,6 +1004,7 @@ public ResponseEntity<BatchJobResponse> launchArchiveStudentsJob(@RequestBody St
response.setStatus(BatchStatusEnum.STARTED.name());

try {
validateInputArchiveStudents(studentSearchRequest);
String searchData = jsonTransformer.marshall(studentSearchRequest);
builder.addString(SEARCH_REQUEST, StringUtils.defaultString(searchData, "{}"));
Job job = jobRegistry.getJob(ARCHIVE_STUDENTS_BATCH_JOB);
Expand All @@ -984,12 +1014,14 @@ public ResponseEntity<BatchJobResponse> launchArchiveStudentsJob(@RequestBody St
ExecutionContext jobContext = jobExecution.getExecutionContext();
DistributionSummaryDTO summaryDTO = new DistributionSummaryDTO();
summaryDTO.setBatchId(jobExecution.getId());
summaryDTO.setUserName(userName);
summaryDTO.setStudentSearchRequest(studentSearchRequest);
jobContext.put(DISDTO, summaryDTO);
response.setBatchId(jobExecution.getId());
} else {
response.setBatchId(jobParameters.getLong("run.id"));
}
return ResponseEntity.ok(response);
return ResponseEntity.status(HttpStatus.CREATED).body(response);
} catch (JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException
| JobParametersInvalidException | NoSuchJobException | IllegalArgumentException e) {
response.setException(e.getLocalizedMessage());
Expand Down
5 changes: 2 additions & 3 deletions .../gov/educ/api/batchgraduation/listener/ArchiveStudentsCompletionNotificationListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import ca.bc.gov.educ.api.batchgraduation.model.DistributionSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest;
import ca.bc.gov.educ.api.batchgraduation.util.DateUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.BatchStatus;
Expand Down Expand Up @@ -49,8 +48,8 @@ public void afterJob(JobExecution jobExecution) {

updateUserSchedulingJobs(jobParameters);

StudentSearchRequest searchRequest = (StudentSearchRequest)jsonTransformer.unmarshall(studentSearchRequest, StudentSearchRequest.class);
String userName = StringUtils.defaultString(jobParameters.getString("runBy"), StringUtils.defaultString(searchRequest.getUser(), "Batch Archive Process"));
StudentSearchRequest searchRequest = summaryDTO.getStudentSearchRequest();
String userName = extractUserName(summaryDTO, jobParameters, searchRequest);

String jobParametersDTO = buildJobParametersDTO(jobType, studentSearchRequest, null, null);
// save batch job & error history
Expand Down
4 changes: 4 additions & 0 deletions .../educ/api/batchgraduation/listener/BaseDistributionRunCompletionNotificationListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,4 +154,8 @@ void updateUserSchedulingJobs(JobParameters jobParameters) {
taskSchedulingService.updateUserScheduledJobs(userScheduledId);
}
}

String extractUserName(BaseSummaryDTO summaryDTO, JobParameters jobParameters, StudentSearchRequest searchRequest) {
return StringUtils.defaultString(StringUtils.defaultString(summaryDTO.getUserName(), jobParameters.getString("runBy")), StringUtils.defaultString(searchRequest.getUser(), "Batch Process"));
}
}
2 changes: 1 addition & 1 deletion ...va/ca/bc/gov/educ/api/batchgraduation/listener/BaseRunCompletionNotificationListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ protected void handleSummary(JobExecution jobExecution, String summaryDtoName, b
if (!isSpecialRun) {
updateBackStudentFlagForErroredStudents(summaryDTO.getErrors(), jobType, obj.getAccess_token());
}
if(!StringUtils.equalsAnyIgnoreCase(req.getActivityCode(), TVRCREATE, TVRUPDATE, TVRDELETE)) {
if(!StringUtils.equalsAnyIgnoreCase(req.getActivityCode(), TVRCREATE, TVRUPDATE, TVRDELETE, ALL)) {
processSchoolList(jobExecutionId, jobType);
}
}
Expand Down
20 changes: 14 additions & 6 deletions ...educ/api/batchgraduation/listener/DeleteStudentReportsCompletionNotificationListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import ca.bc.gov.educ.api.batchgraduation.model.DistributionSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest;
import ca.bc.gov.educ.api.batchgraduation.util.DateUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.BatchStatus;
Expand All @@ -13,6 +12,8 @@
import org.springframework.stereotype.Component;

import java.util.Date;
import java.util.List;
import java.util.UUID;

import static ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants.SEARCH_REQUEST;

Expand Down Expand Up @@ -49,16 +50,23 @@ public void afterJob(JobExecution jobExecution) {
updateUserSchedulingJobs(jobParameters);

StudentSearchRequest searchRequest = summaryDTO.getStudentSearchRequest();
String userName = StringUtils.defaultString(jobParameters.getString("runBy"), StringUtils.defaultString(searchRequest.getUser(), "Batch TVR Delete Process"));
String userName = extractUserName(summaryDTO, jobParameters, searchRequest);

summaryDTO.getSchools().forEach((value) -> LOGGER.info("School {} number of Deleted Student Reports : {}", value.getMincode(), value.getNumberOfStudents()));
if(summaryDTO.getProcessedCount() > 0) {
List<UUID> finalStudentGuids = searchRequest.getStudentIDs();
int partitionSize = finalStudentGuids.size()/200;
partitionSize = partitionSize == 0 ? finalStudentGuids.size() : partitionSize;
for (int i = 0; i < finalStudentGuids.size(); i += partitionSize) {
List<UUID> studentGuidsSubList = finalStudentGuids.subList(i, Math.min(i + partitionSize, finalStudentGuids.size()));
restUtils.updateStudentGradRecordHistory(studentGuidsSubList, jobExecutionId, userName, "TVRDELETED");
}
}
String jobParametersDTO = buildJobParametersDTO(jobType, studentSearchRequest, null, null);
// save batch job & error history
processBatchJobHistory(summaryDTO, jobExecutionId, status, jobTrigger, jobType, startTime, endTime, jobParametersDTO);
LOGGER.info(" --------------------------------------------------------------------------------------");
summaryDTO.getSchools().forEach((value) -> LOGGER.info("School {} number of Deleted Student Reports : {}", value.getMincode(), value.getNumberOfStudents()));
if(summaryDTO.getProcessedCount() > 0) {
restUtils.updateStudentGradRecordHistory(searchRequest.getStudentIDs(), jobExecutionId, userName, "TVRDELETED");
}

}
}
}
2 changes: 2 additions & 0 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/model/BaseSummaryDTO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,6 @@ public class BaseSummaryDTO implements Serializable {

@JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
private String accessToken;

private String userName;
}
4 changes: 2 additions & 2 deletions ...main/java/ca/bc/gov/educ/api/batchgraduation/processor/DeleteStudentReportsProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ public class DeleteStudentReportsProcessor implements ItemProcessor<List<UUID>,
@Override
public List<UUID> process(List<UUID> uuids) throws Exception {
Long batchId = summaryDTO.getBatchId();
StudentSearchRequest searchRequest = summaryDTO.getStudentSearchRequest();
if(LOGGER.isDebugEnabled()) {
LOGGER.debug("Process Student Reports: {}", uuids.size());
LOGGER.debug("Process Student Reports: {}", uuids.isEmpty() ? searchRequest.getActivityCode() : uuids.size());
}
long countDeletedStudentReports = summaryDTO.getProcessedCount();
StudentSearchRequest searchRequest = summaryDTO.getStudentSearchRequest();
for(String reportType: searchRequest.getReportTypes()) {
countDeletedStudentReports += restUtils.deleteStudentReports(batchId, uuids, reportType, summaryDTO);
}
Expand Down
12 changes: 7 additions & 5 deletions .../main/java/ca/bc/gov/educ/api/batchgraduation/reader/ArchiveSchoolReportsPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ public JobExecution getJobExecution() {

@Override
public Map<String, ExecutionContext> partition(int gridSize) {
DistributionSummaryDTO summaryDTO = (DistributionSummaryDTO)jobExecution.getExecutionContext().get("distributionSummaryDTO");
if(summaryDTO == null) {
summaryDTO = new DistributionSummaryDTO();
jobExecution.getExecutionContext().put("distributionSummaryDTO", summaryDTO);
}

StudentSearchRequest searchRequest = getStudentSearchRequest();
long startTime = System.currentTimeMillis();
logger.debug("Filter Schools for archiving school reports");
Expand All @@ -47,13 +53,9 @@ public Map<String, ExecutionContext> partition(int gridSize) {
logger.debug("Final list of eligible District / School codes {}", String.join(", ", finalSchoolDistricts));
}

DistributionSummaryDTO summaryDTO = (DistributionSummaryDTO)jobExecution.getExecutionContext().get("distributionSummaryDTO");
if(summaryDTO == null) {
summaryDTO = new DistributionSummaryDTO();
jobExecution.getExecutionContext().put("distributionSummaryDTO", summaryDTO);
}
summaryDTO.setBatchId(jobExecution.getId());
summaryDTO.setStudentSearchRequest(searchRequest);

Long totalSchoolReporsCount = 0L;
List<String> reportTypes = searchRequest.getReportTypes();
Long schoolReportsCount = 0L;
Expand Down
12 changes: 7 additions & 5 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/ArchiveStudentsPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ public JobExecution getJobExecution() {

@Override
public Map<String, ExecutionContext> partition(int gridSize) {
DistributionSummaryDTO summaryDTO = (DistributionSummaryDTO)jobExecution.getExecutionContext().get("distributionSummaryDTO");
if(summaryDTO == null) {
summaryDTO = new DistributionSummaryDTO();
jobExecution.getExecutionContext().put("distributionSummaryDTO", summaryDTO);
}

StudentSearchRequest searchRequest = getStudentSearchRequest();
long startTime = System.currentTimeMillis();
logger.debug("Filter Schools for archiving students");
Expand All @@ -48,13 +54,9 @@ public Map<String, ExecutionContext> partition(int gridSize) {
logger.debug("Final list of eligible District / School codes {}", String.join(", ", finalSchoolDistricts));
}

DistributionSummaryDTO summaryDTO = (DistributionSummaryDTO)jobExecution.getExecutionContext().get("distributionSummaryDTO");
if(summaryDTO == null) {
summaryDTO = new DistributionSummaryDTO();
jobExecution.getExecutionContext().put("distributionSummaryDTO", summaryDTO);
}
summaryDTO.setBatchId(jobExecution.getId());
summaryDTO.setStudentSearchRequest(searchRequest);

List<String> studentStatusCodes = searchRequest.getStatuses();
Long totalStudentsCount = 0L;
for(String schoolOfRecord: finalSchoolDistricts) {
Expand Down
12 changes: 8 additions & 4 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BasePartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,21 @@ protected void saveInputData(List<UUID> studentIDs) {

protected BatchGradAlgorithmJobHistoryEntity createBatchJobHistory() {
Long jobExecutionId = getJobExecution().getId();
JobParameters jobParameters = getJobExecution().getJobParameters();
return createBatchJobHistory(jobExecutionId, getJobExecution(), 0);
}

protected BatchGradAlgorithmJobHistoryEntity createBatchJobHistory(Long jobExecutionId, JobExecution jobExecution, long expectedStudentsProcessed) {
JobParameters jobParameters = jobExecution.getJobParameters();
String jobTrigger = jobParameters.getString("jobTrigger");
String jobType = jobParameters.getString("jobType");
String username = jobParameters.getString(RUN_BY);
String studentSearchRequest = jobParameters.getString(SEARCH_REQUEST);
String status = getJobExecution().getStatus().toString();
Date startTime = DateUtils.toDate(getJobExecution().getStartTime());
String status = jobExecution.getStatus().toString();
Date startTime = DateUtils.toDate(jobExecution.getStartTime());

BatchGradAlgorithmJobHistoryEntity ent = new BatchGradAlgorithmJobHistoryEntity();
ent.setActualStudentsProcessed(0L);
ent.setExpectedStudentsProcessed(0L);
ent.setExpectedStudentsProcessed(expectedStudentsProcessed);
ent.setFailedStudentsProcessed(0);
ent.setJobExecutionId(jobExecutionId);
ent.setStartTime(DateUtils.toLocalDateTime(startTime));
Expand Down
Loading

0 comments on commit f7ffa4e

Please sign in to comment.