Implementing pagination for _cat/indices API

plugins/examples/rest-handler/src/main/java/org/opensearch/example/resthandler/ExampleCatAction.java

@@ -83,7 +83,7 @@ protected RestChannelConsumer doCatRequest(final RestRequest request, final Node
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append(documentation());
     }
 

server/src/main/java/org/opensearch/action/ActionModule.java

@@ -459,6 +459,8 @@
 import org.opensearch.rest.action.ingest.RestGetPipelineAction;
 import org.opensearch.rest.action.ingest.RestPutPipelineAction;
 import org.opensearch.rest.action.ingest.RestSimulatePipelineAction;
+import org.opensearch.rest.action.list.RestIndicesListAction;
+import org.opensearch.rest.action.list.RestListAction;
 import org.opensearch.rest.action.search.RestClearScrollAction;
 import org.opensearch.rest.action.search.RestCountAction;
 import org.opensearch.rest.action.search.RestCreatePitAction;
@@ -799,9 +801,14 @@ private ActionFilters setupActionFilters(List<ActionPlugin> actionPlugins) {
 
     public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         List<AbstractCatAction> catActions = new ArrayList<>();
+        List<AbstractCatAction> listActions = new ArrayList<>();
         Consumer<RestHandler> registerHandler = handler -> {
             if (handler instanceof AbstractCatAction) {
-                catActions.add((AbstractCatAction) handler);
+                if (((AbstractCatAction) handler).isActionPaginated()) {
+                    listActions.add((AbstractCatAction) handler);
+                } else {
+                    catActions.add((AbstractCatAction) handler);
+                }
             }
             restController.registerHandler(handler);
         };
@@ -977,6 +984,9 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         }
         registerHandler.accept(new RestTemplatesAction());
 
+        // LIST API
+        registerHandler.accept(new RestIndicesListAction());
+
         // Point in time API
         registerHandler.accept(new RestCreatePitAction());
         registerHandler.accept(new RestDeletePitAction());
@@ -1008,6 +1018,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
             }
         }
         registerHandler.accept(new RestCatAction(catActions));
+        registerHandler.accept(new RestListAction(listActions));
         registerHandler.accept(new RestDecommissionAction());
         registerHandler.accept(new RestGetDecommissionStateAction());
         registerHandler.accept(new RestRemoteStoreStatsAction());

server/src/main/java/org/opensearch/common/Table.java

@@ -34,6 +34,7 @@
 
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.core.common.Strings;
+import org.opensearch.rest.pagination.PaginatedQueryResponse;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
@@ -59,9 +60,19 @@ public class Table {
     private List<Cell> currentCells;
     private boolean inHeaders = false;
     private boolean withTime = false;
+    /**
+     * paginatedQueryResponse if null will imply the Table response is not paginated.
+     */
+    private PaginatedQueryResponse paginatedQueryResponse;
     public static final String EPOCH = "epoch";
     public static final String TIMESTAMP = "timestamp";
 
+    public Table() {}
+
+    public Table(@Nullable PaginatedQueryResponse paginatedQueryResponse) {
+        this.paginatedQueryResponse = paginatedQueryResponse;
+    }
+
     public Table startHeaders() {
         inHeaders = true;
         currentCells = new ArrayList<>();
@@ -230,6 +241,10 @@ public Map<String, String> getAliasMap() {
         return headerAliasMap;
     }
 
+    public PaginatedQueryResponse getPaginatedQueryResponse() {
+        return paginatedQueryResponse;
+    }
+
     /**
      * Cell in a table
      *

server/src/main/java/org/opensearch/rest/RestRequest.java

@@ -51,6 +51,7 @@
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.http.HttpChannel;
 import org.opensearch.http.HttpRequest;
+import org.opensearch.rest.pagination.PaginatedQueryRequest;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -67,6 +68,9 @@
 
 import static org.opensearch.common.unit.TimeValue.parseTimeValue;
 import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
+import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY;
+import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_SIZE_KEY;
+import static org.opensearch.rest.pagination.PaginatedQueryRequest.PAGINATED_QUERY_PARAM_SORT_KEY;
 
 /**
  * REST Request
@@ -591,6 +595,14 @@ public static MediaType parseContentType(List<String> header) {
         throw new IllegalArgumentException("empty Content-Type header");
     }
 
+    public PaginatedQueryRequest parsePaginatedQueryParams(String defaultSortOrder, int defaultPageSize) {
+        return new PaginatedQueryRequest(
+            param(PAGINATED_QUERY_PARAM_NEXT_TOKEN_KEY),
+            param(PAGINATED_QUERY_PARAM_SORT_KEY, defaultSortOrder),
+            paramAsInt(PAGINATED_QUERY_PARAM_SIZE_KEY, defaultPageSize)
+        );
+    }
+
     /**
      * Thrown if there is an error in the content type header.
      *

server/src/main/java/org/opensearch/rest/action/cat/AbstractCatAction.java

@@ -40,11 +40,13 @@
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.BytesRestResponse;
 import org.opensearch.rest.RestRequest;
+import org.opensearch.rest.pagination.PaginatedQueryRequest;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.opensearch.rest.action.cat.RestTable.buildHelpWidths;
@@ -57,9 +59,11 @@
  */
 public abstract class AbstractCatAction extends BaseRestHandler {
 
+    protected PaginatedQueryRequest paginatedQueryRequest;
+
     protected abstract RestChannelConsumer doCatRequest(RestRequest request, NodeClient client);
 
-    protected abstract void documentation(StringBuilder sb);
+    public abstract void documentation(StringBuilder sb);
 
     protected abstract Table getTableWithHeader(RestRequest request);
 
@@ -85,6 +89,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
                 channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, bytesOutput.bytes()));
             };
         } else {
+            if (isActionPaginated()) {
+                this.paginatedQueryRequest = validateAndGetPaginationMetadata(request);
+                assert Objects.nonNull(paginatedQueryRequest) : "paginatedQueryRequest can not be null for paginated queries";
+            }
             return doCatRequest(request, client);
         }
     }
@@ -98,4 +106,23 @@ protected Set<String> responseParams() {
         return RESPONSE_PARAMS;
     }
 
+    /**
+     *
+     * @return boolean denoting whether the RestAction will output paginated responses or not.
+     * Is kept false by default, every paginated action to override and return true.
+     */
+    public boolean isActionPaginated() {
+        return false;
+    }
+
+    /**
+     *
+     * @return Metadata that can be extracted out from the rest request. Each paginated action to override and provide
+     * its own implementation. Query params supported by the action specific to pagination along with the respective validations,
+     * should be added here.
+     */
+    protected PaginatedQueryRequest validateAndGetPaginationMetadata(RestRequest restRequest) {
+        return null;
+    }
+
 }

server/src/main/java/org/opensearch/rest/action/cat/RestAliasAction.java

@@ -89,7 +89,7 @@ public RestResponse buildResponse(GetAliasesResponse response) throws Exception
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/aliases\n");
         sb.append("/_cat/aliases/{alias}\n");
     }

server/src/main/java/org/opensearch/rest/action/cat/RestAllocationAction.java

@@ -78,7 +78,7 @@ public String getName() {
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/allocation\n");
     }
 

server/src/main/java/org/opensearch/rest/action/cat/RestCatRecoveryAction.java

@@ -77,7 +77,7 @@ public String getName() {
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/recovery\n");
         sb.append("/_cat/recovery/{index}\n");
     }

server/src/main/java/org/opensearch/rest/action/cat/RestCatSegmentReplicationAction.java

@@ -58,7 +58,7 @@ public String getName() {
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/segment_replication\n");
         sb.append("/_cat/segment_replication/{index}\n");
     }

server/src/main/java/org/opensearch/rest/action/cat/RestClusterManagerAction.java

@@ -69,7 +69,7 @@ public String getName() {
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/cluster_manager\n");
     }
 

server/src/main/java/org/opensearch/rest/action/cat/RestCountAction.java

@@ -71,7 +71,7 @@ public String getName() {
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/count\n");
         sb.append("/_cat/count/{index}\n");
     }

server/src/main/java/org/opensearch/rest/action/cat/RestFielddataAction.java

@@ -82,7 +82,7 @@ public RestResponse buildResponse(NodesStatsResponse nodeStatses) throws Excepti
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/fielddata\n");
         sb.append("/_cat/fielddata/{fields}\n");
     }

server/src/main/java/org/opensearch/rest/action/cat/RestHealthAction.java

@@ -69,7 +69,7 @@ public boolean allowSystemIndexAccessByDefault() {
     }
 
     @Override
-    protected void documentation(StringBuilder sb) {
+    public void documentation(StringBuilder sb) {
         sb.append("/_cat/health\n");
     }