diff --git a/design/BulkAPI.md b/design/BulkAPI.md new file mode 100644 index 000000000..19f129ede --- /dev/null +++ b/design/BulkAPI.md @@ -0,0 +1,159 @@ +# Bulk API Documentation + +Bulk is an API designed to provide resource optimization recommendations in bulk for all available +containers, namespaces, etc., for a cluster connected via the datasource integration framework. Bulk can +be configured using filters like exclude/include namespaces, workloads, containers, or labels for generating +recommendations. It also has settings to generate recommendations at both the container or namespace level, or both. + +Bulk returns a `jobID` as a response to track the job status. The user can use the `jobID` to monitor the +progress of the job. + +## Task Flow When Bulk Is Invoked + +1. Returns a unique `jobID`. +2. Background Bulk: + - First, does a handshake with the datasource. + - Using queries, it fetches the list of namespaces, workloads, containers of the connected datasource. + - Creates experiments, one for each container *alpha release. + - Triggers `generateRecommendations` for each container. + - Once all experiments are created, and recommendations are generated, the system marks the `jobID` as "COMPLETED". + +## API Specification + +### POST /bulk + +**Request Payload (JSON):** + +```json +{ + "filter": { + "exclude": { + "namespace": [], + "workload": [], + "containers": [], + "labels": {} + }, + "include": { + "namespace": [], + "workload": [], + "containers": [], + "labels": { + "key1": "value1", + "key2": "value2" + } + } + }, + "time_range": { + "start": "", + "end": "" + }, + "datasource": "Cbank1Xyz", + "experiment_types": [ + "container", + "namespace" + ] +} +``` + +**filter:** This object contains both exclusion and inclusion filters to specify the scope of data being queried. + +- **exclude:** Defines the criteria to exclude certain data. + - **namespace:** A list of Kubernetes namespaces to exclude. If empty, no namespaces are excluded. + - **workload:** A list of workloads to exclude. + - **containers:** A list of container names to exclude. + - **labels:** Key-value pairs of labels to exclude. + +- **include:** Defines the criteria to include specific data. + - **namespace:** A list of Kubernetes namespaces to include. + - **workload:** A list of workloads to include. + - **containers:** A list of container names to include. + - **labels:** Key-value pairs of labels to include. + +- **time_range:** Specifies the time range for querying the data. If empty, no specific time range is applied. + +- **datasource:** The data source, e.g., `"Cbank1Xyz"`. + +- **experiment_types:** Specifies the type(s) of experiments to run, e.g., `"container"` or `"namespace"`. + +### Success Response + +- **Status:** 200 OK +- **Body:** + +```json +{ + "jobid": "123e4567-e89b-12d3-a456-426614174000" +} +``` + +### GET Request: + +```bash +GET /bulk?jobid=123e4567-e89b-12d3-a456-426614174000 +``` + +**Body (JSON):** + +```json +{ + "jobID": "123e4567-e89b-12d3-a456-426614174000", + "status": "IN-PROGRESS", + "progress": 30, + "data": { + "experiments": { + "new": [ + "a", + "b", + "c" + ], + "updated": [], + "failed": [] + }, + "recommendations": { + "count": 9, + "completed": 3, + "experiments": { + "completed": [ + "exp1", + "exp2", + "exp3" + ], + "progress": [ + "exp1", + "exp2", + "exp3" + ], + "new": [ + "exp1", + "exp2", + "exp3" + ], + "failed": [] + } + } + }, + "job_start_time": "2024-09-23T10:58:47.048Z", + "job_end_time": "2024-09-23T11:01:52.205Z" +} +``` + +### Response Parameters + +- **jobID:** Unique identifier for the job. +- **status:** Current status of the job. Possible values: `"IN-PROGRESS"`, `"COMPLETED"`, `"FAILED"`. +- **progress:** Percentage of job completion. +- **data:** Contains detailed information about the experiments and recommendations. + - **experiments:** Tracks the status of experiments. + - **new:** List of newly created experiments. + - **updated:** List of updated experiments. + - **failed:** List of experiments that failed. + - **recommendations:** Provides details on recommendations. + - **count:** Total number of recommendations. + - **completed:** Number of completed recommendations. + - **experiments:** + - **completed:** List of experiments with completed recommendations. + - **progress:** List of experiments in progress. + - **new:** List of new experiments. + - **failed:** List of failed experiments. +- **job_start_time:** Timestamp indicating when the job started. +- **job_end_time:** Timestamp indicating when the job finished. diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java b/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java new file mode 100644 index 000000000..185b19679 --- /dev/null +++ b/src/main/java/com/autotune/analyzer/serviceObjects/BulkInput.java @@ -0,0 +1,121 @@ +package com.autotune.analyzer.serviceObjects; + +import java.util.List; +import java.util.Map; + +public class BulkInput { + private FilterWrapper filter; + private TimeRange time_range; + private String datasource; + + // Getters and Setters + + public TimeRange getTime_range() { + return time_range; + } + + public void setTime_range(TimeRange time_range) { + this.time_range = time_range; + } + + public String getDatasource() { + return datasource; + } + + public void setDatasource(String datasource) { + this.datasource = datasource; + } + + public FilterWrapper getFilter() { + return filter; + } + + public void setFilter(FilterWrapper filter) { + this.filter = filter; + } + + // Nested class for FilterWrapper that contains 'exclude' and 'include' + public static class FilterWrapper { + private Filter exclude; + private Filter include; + + // Getters and Setters + public Filter getExclude() { + return exclude; + } + + public void setExclude(Filter exclude) { + this.exclude = exclude; + } + + public Filter getInclude() { + return include; + } + + public void setInclude(Filter include) { + this.include = include; + } + } + + public static class Filter { + private List namespace; + private List workload; + private List containers; + private Map labels; + + // Getters and Setters + public List getNamespace() { + return namespace; + } + + public void setNamespace(List namespace) { + this.namespace = namespace; + } + + public List getWorkload() { + return workload; + } + + public void setWorkload(List workload) { + this.workload = workload; + } + + public List getContainers() { + return containers; + } + + public void setContainers(List containers) { + this.containers = containers; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + } + + public static class TimeRange { + private String start; + private String end; + + // Getters and Setters + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = start; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + } +} diff --git a/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java b/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java new file mode 100644 index 000000000..fe4b313a7 --- /dev/null +++ b/src/main/java/com/autotune/analyzer/serviceObjects/BulkJobStatus.java @@ -0,0 +1,265 @@ +package com.autotune.analyzer.serviceObjects; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; + +public class BulkJobStatus { + private String jobID; + private String status; + private int progress; + private Data data; + @JsonProperty("start_time") + private String startTime; // Change to String to store formatted time + @JsonProperty("end_time") + private String endTime; // Change to String to store formatted time + + public BulkJobStatus(String jobID, String status, int progress, Data data, Instant startTime) { + this.jobID = jobID; + this.status = status; + this.progress = progress; + this.data = data; + setStartTime(startTime); + } + + public String getJobID() { + return jobID; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public int getProgress() { + return progress; + } + + public void setProgress(int progress) { + this.progress = progress; + } + + public Data getData() { + return data; + } + + public void setData(Data data) { + this.data = data; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(Instant startTime) { + this.startTime = formatInstantAsUTCString(startTime); + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(Instant endTime) { + this.endTime = formatInstantAsUTCString(endTime); + } + + // Utility function to format Instant into the required UTC format + private String formatInstantAsUTCString(Instant instant) { + DateTimeFormatter formatter = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") + .withZone(ZoneOffset.UTC); // Ensure it's in UTC + + return formatter.format(instant); + } + + // Inner class for the data field + public static class Data { + private Experiments experiments; + private Recommendations recommendations; + + public Data(Experiments experiments, Recommendations recommendations) { + this.experiments = experiments; + this.recommendations = recommendations; + } + + public Experiments getExperiments() { + return experiments; + } + + public void setExperiments(Experiments experiments) { + this.experiments = experiments; + } + + public Recommendations getRecommendations() { + return recommendations; + } + + public void setRecommendations(Recommendations recommendations) { + this.recommendations = recommendations; + } + } + + // Inner class for experiments + public static class Experiments { + @JsonProperty("new") + private List newExperiments; + @JsonProperty("updated") + private List updatedExperiments; + @JsonProperty("failed") + private List failedExperiments; + + public Experiments(List newExperiments, List updatedExperiments) { + this.newExperiments = newExperiments; + this.updatedExperiments = updatedExperiments; + } + + public List getNewExperiments() { + return newExperiments; + } + + public void setNewExperiments(List newExperiments) { + this.newExperiments = newExperiments; + } + + public List getUpdatedExperiments() { + return updatedExperiments; + } + + public void setUpdatedExperiments(List updatedExperiments) { + this.updatedExperiments = updatedExperiments; + } + } + + // Inner class for recommendations + public static class Recommendations { + @JsonProperty("count") + private int totalCount; + @JsonProperty("completed") + private int completedCount; + private RecommendationData data; + + public Recommendations(int totalCount, int completedCount, RecommendationData data) { + this.totalCount = totalCount; + this.completedCount = completedCount; + this.data = data; + } + + public int getTotalCount() { + return totalCount; + } + + public void setTotalCount(int totalCount) { + this.totalCount = totalCount; + } + + public int getCompletedCount() { + return this.data.getCompleted().size(); + } + + public void setCompletedCount(int completedCount) { + this.completedCount = completedCount; + } + + public RecommendationData getData() { + return data; + } + + public void setData(RecommendationData data) { + this.data = data; + } + } + + // Inner class for recommendation data + public static class RecommendationData { + private List completed; + private List progress; + private List inqueue; + private List failed; + + public RecommendationData(List completed, List progress, List inqueue, List failed) { + this.completed = completed; + this.progress = progress; + this.inqueue = inqueue; + this.failed = failed; + } + + public List getCompleted() { + return completed; + } + + public void setCompleted(List completed) { + this.completed = completed; + } + + public List getProgress() { + return progress; + } + + public void setProgress(List progress) { + this.progress = progress; + } + + public List getInqueue() { + return inqueue; + } + + public void setInqueue(List inqueue) { + this.inqueue = inqueue; + } + + public List getFailed() { + return failed; + } + + public void setFailed(List failed) { + this.failed = failed; + } + + // Move elements from inqueue to progress + public void moveToProgress(String element) { + if (inqueue.contains(element)) { + inqueue.remove(element); + if (!progress.contains(element)) { + progress.add(element); + } + } + } + + // Move elements from progress to completed + public void moveToCompleted(String element) { + if (progress.contains(element)) { + progress.remove(element); + if (!completed.contains(element)) { + completed.add(element); + } + } + } + + // Move elements from progress to failed + public void moveToFailed(String element) { + if (progress.contains(element)) { + progress.remove(element); + if (!failed.contains(element)) { + failed.add(element); + } + } + } + + // Calculate the percentage of completion + public int completionPercentage() { + int totalTasks = completed.size() + progress.size() + inqueue.size() + failed.size(); + if (totalTasks == 0) { + return (int) 0.0; + } + return (int) ((completed.size() * 100.0) / totalTasks); + } + } + + +}