Skip to content

Commit

Permalink
Fix #141 [skip ci]
Browse files Browse the repository at this point in the history
  • Loading branch information
hmiguim committed Oct 24, 2019
1 parent bad1a50 commit f19c75c
Show file tree
Hide file tree
Showing 51 changed files with 936 additions and 679 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import com.databasepreservation.common.api.common.ConsumesOutputStream;
import com.databasepreservation.common.api.common.EntityResponse;

import javax.ws.rs.core.StreamingOutput;

/**
* @author Bruno Ferreira <bferreira@keep.pt>
*/
Expand All @@ -15,6 +17,13 @@ public class StreamResponse implements EntityResponse {
private String mediaType;
private ConsumesOutputStream stream;

public StreamResponse(ConsumesOutputStream stream) {
this.filename = stream.getFileName();
this.mediaType = stream.getMediaType();
this.mediaType = stream.getMediaType();
this.stream = stream;
}

public StreamResponse(String filename, String mediaType, ConsumesOutputStream stream) {
super();
this.filename = filename;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package com.databasepreservation.common.api.utils;

import com.databasepreservation.common.api.common.ConsumesOutputStream;

import javax.ws.rs.core.StreamingOutput;
import java.io.IOException;
import java.io.OutputStream;

/**
* @author Miguel Guimarães <mguimaraes@keep.pt>
*/
public class ViewerStreamingOutput implements StreamingOutput {
private final ConsumesOutputStream outputHandler;

public ViewerStreamingOutput(final ConsumesOutputStream outputHandler) {
this.outputHandler = outputHandler;
}

@Override
public void write(final OutputStream output) throws IOException {
outputHandler.consumeOutputStream(output);
}

public StreamResponse toStreamResponse() {
return new StreamResponse(outputHandler);
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.databasepreservation.common.api.v1;

import java.io.InputStream;
import java.util.List;

import javax.servlet.http.HttpServletRequest;
Expand All @@ -16,19 +15,25 @@
import org.apache.commons.lang3.StringUtils;
import org.roda.core.data.exceptions.RODAException;
import org.roda.core.data.utils.JsonUtils;
import org.roda.core.data.v2.index.IndexResult;
import org.roda.core.data.v2.index.filter.Filter;
import org.roda.core.data.v2.index.sort.Sorter;
import org.roda.core.data.v2.index.sublist.Sublist;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import com.databasepreservation.common.server.ViewerFactory;
import com.databasepreservation.common.api.utils.ApiUtils;
import com.databasepreservation.common.api.utils.DownloadUtils;
import com.databasepreservation.common.api.utils.StreamResponse;
import com.databasepreservation.common.api.utils.ViewerStreamingOutput;
import com.databasepreservation.common.api.v1.utils.IterableIndexResultsCSVOutputStream;
import com.databasepreservation.common.api.v1.utils.ResultsCSVOutputStream;
import com.databasepreservation.common.server.ViewerFactory;
import com.databasepreservation.common.server.index.DatabaseRowsSolrManager;
import com.databasepreservation.common.server.index.utils.IterableIndexResult;
import com.databasepreservation.common.shared.ViewerConstants;
import com.databasepreservation.common.shared.ViewerStructure.ViewerDatabase;
import com.databasepreservation.common.shared.ViewerStructure.ViewerRow;
import com.databasepreservation.common.shared.ViewerStructure.ViewerTable;
import com.databasepreservation.common.utils.UserUtility;

import io.swagger.annotations.Api;
Expand Down Expand Up @@ -59,7 +64,10 @@ public Response getCSVResultsPost(@PathParam(ViewerConstants.API_PATH_PARAM_DATA
@QueryParam(ViewerConstants.API_QUERY_PARAM_FILTER) String filterParam,
@QueryParam(ViewerConstants.API_QUERY_PARAM_FIELDS) String fieldsListParam,
@QueryParam(ViewerConstants.API_QUERY_PARAM_SORTER) String sorterParam,
@QueryParam(ViewerConstants.API_QUERY_PARAM_SUBLIST) String subListParam) throws RODAException {
@QueryParam(ViewerConstants.API_QUERY_PARAM_SUBLIST) String subListParam,
@QueryParam(ViewerConstants.API_PATH_PARAM_FILENAME) String filename,
@QueryParam(ViewerConstants.API_PATH_PARAM_EXPORT_DESCRIPTION) Boolean exportDescriptions,
@QueryParam(ViewerConstants.API_PATH_PARAM_TABLE_UUID) String tableUUID) throws RODAException {
DatabaseRowsSolrManager solrManager = ViewerFactory.getSolrManager();

Filter filter = JsonUtils.getObjectFromJson(filterParam, Filter.class);
Expand All @@ -72,9 +80,20 @@ public Response getCSVResultsPost(@PathParam(ViewerConstants.API_PATH_PARAM_DATA

UserUtility.Authorization.checkDatabaseAccessPermission(this.request, databaseUUID);

InputStream rowsCSV = solrManager.findRowsCSV(databaseUUID, filter, sorter, sublist, fields);
final ViewerDatabase database = solrManager.retrieve(ViewerDatabase.class, databaseUUID);
final ViewerTable table = database.getMetadata().getTable(tableUUID);

return ApiUtils
.okResponse(new StreamResponse("file.csv", MediaType.APPLICATION_OCTET_STREAM, DownloadUtils.stream(rowsCSV)));
if (sublist == null) {
final IterableIndexResult allRows = solrManager.findAllRows(databaseUUID, filter, sorter, fields);
return ApiUtils.okResponse(new ViewerStreamingOutput(
new IterableIndexResultsCSVOutputStream(allRows, table, fields, filename, exportDescriptions, ','))
.toStreamResponse());
} else {
final IndexResult<ViewerRow> rows = solrManager.findRows(databaseUUID, filter, sorter, sublist, null, fields);

return ApiUtils.okResponse(
new ViewerStreamingOutput(new ResultsCSVOutputStream(rows, table, fields, filename, exportDescriptions, ','))
.toStreamResponse());
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package com.databasepreservation.common.api.v1.utils;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.Iterator;
import java.util.List;

import com.databasepreservation.common.server.index.utils.IterableIndexResult;
import org.apache.commons.csv.CSVPrinter;
import org.roda.core.data.v2.index.IndexResult;

import com.databasepreservation.common.shared.ViewerStructure.ViewerRow;
import com.databasepreservation.common.shared.ViewerStructure.ViewerTable;

/**
* @author Miguel Guimarães <mguimaraes@keep.pt>
*/
public class IterableIndexResultsCSVOutputStream extends CSVOutputStream {
/** The results to write to output stream. */
private final IterableIndexResult results;
private final ViewerTable table;
private final List<String> fieldsToReturn;
private final boolean exportDescription;
/**
* Constructor.
*
* @param results
* the results to write to output stream.
* @param filename
* the filename.
* @param delimiter
* the CSV field delimiter.
*/
public IterableIndexResultsCSVOutputStream(final IterableIndexResult results, final ViewerTable table, final List<String> fieldsToReturn, final String filename,
final boolean exportDescription, final char delimiter) {
super(filename, delimiter);
this.results = results;
this.table = table;
this.fieldsToReturn = fieldsToReturn;
this.exportDescription = exportDescription;
}

@Override
public void consumeOutputStream(final OutputStream out) throws IOException {
final OutputStreamWriter writer = new OutputStreamWriter(out);
CSVPrinter printer = null;
boolean isFirst = true;

for (ViewerRow row : results) {
if (isFirst) {
printer = getFormat().withHeader(table.getCSVHeaders(fieldsToReturn, exportDescription).toArray(new String[0])).print(writer);
isFirst = false;
}

printer.printRecord(row.getCellValues(fieldsToReturn));

}

writer.flush();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,23 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.List;

import org.apache.commons.csv.CSVPrinter;
import org.roda.core.data.v2.index.IndexResult;

import com.databasepreservation.common.shared.ViewerStructure.IsIndexed;
import com.databasepreservation.common.shared.ViewerStructure.ViewerRow;
import com.databasepreservation.common.shared.ViewerStructure.ViewerTable;

/**
* @author Miguel Guimarães <mguimaraes@keep.pt>
*/
public class ResultsCSVOutputStream<T extends IsIndexed> extends CSVOutputStream {
public class ResultsCSVOutputStream extends CSVOutputStream {
/** The results to write to output stream. */
private final IndexResult<T> results;

private final IndexResult<ViewerRow> results;
private final ViewerTable table;
private final List<String> fieldsToReturn;
private final boolean exportDescription;
/**
* Constructor.
*
Expand All @@ -26,22 +30,27 @@ public class ResultsCSVOutputStream<T extends IsIndexed> extends CSVOutputStream
* @param delimiter
* the CSV field delimiter.
*/
public ResultsCSVOutputStream(final IndexResult<T> results, final String filename, final char delimiter) {
public ResultsCSVOutputStream(final IndexResult<ViewerRow> results, final ViewerTable table, final List<String> fieldsToReturn, final String filename,
final boolean exportDescription, final char delimiter) {
super(filename, delimiter);
this.results = results;
this.table = table;
this.fieldsToReturn = fieldsToReturn;
this.exportDescription = exportDescription;
}

@Override
public void consumeOutputStream(final OutputStream out) throws IOException {
final OutputStreamWriter writer = new OutputStreamWriter(out);
CSVPrinter printer = null;
boolean isFirst = true;
for (final T result : this.results.getResults()) {
for (final ViewerRow row : this.results.getResults()) {
if (isFirst) {
printer = getFormat().withHeader(result.toCsvHeaders().toArray(new String[0])).print(writer);
printer = getFormat().withHeader(table.getCSVHeaders(fieldsToReturn, exportDescription).toArray(new String[0])).print(writer);
isFirst = false;
}
printer.printRecord(result.toCsvValues());

printer.printRecord(row.getCellValues(fieldsToReturn));
}
writer.flush();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package com.databasepreservation.common.server.index;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import com.databasepreservation.common.server.index.schema.collections.DatabasesCollection;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
Expand All @@ -28,10 +27,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.databasepreservation.common.exceptions.ViewerException;
import com.databasepreservation.common.server.index.schema.SolrCollection;
import com.databasepreservation.common.server.index.schema.SolrDefaultCollectionRegistry;
import com.databasepreservation.common.server.index.schema.SolrRowsCollectionRegistry;
import com.databasepreservation.common.server.index.schema.collections.RowsCollection;
import com.databasepreservation.common.server.index.utils.IterableIndexResult;
import com.databasepreservation.common.server.index.utils.JsonTransformer;
import com.databasepreservation.common.server.index.utils.Pair;
import com.databasepreservation.common.server.index.utils.SolrUtils;
Expand All @@ -43,7 +44,6 @@
import com.databasepreservation.common.shared.ViewerStructure.ViewerRow;
import com.databasepreservation.common.shared.ViewerStructure.ViewerTable;
import com.databasepreservation.common.shared.client.common.search.SavedSearch;
import com.databasepreservation.common.exceptions.ViewerException;
import com.databasepreservation.utils.FileUtils;

/**
Expand Down Expand Up @@ -77,10 +77,10 @@ public void addDatabaseMetadata(ViewerDatabase database) throws ViewerException
insertDocument(ViewerDatabase.class, database);
}

public void addDatabaseRowCollection(ViewerDatabase database) throws ViewerException {
updateValidationFields(database.getUUID(),
public void addDatabaseRowCollection(final String databaseUUID) throws ViewerException {
updateValidationFields(databaseUUID,
Pair.of(ViewerConstants.SOLR_DATABASES_STATUS, ViewerDatabase.Status.INGESTING.toString()));
RowsCollection collection = new RowsCollection(database.getUUID());
RowsCollection collection = new RowsCollection(databaseUUID);
collection.createRowsCollection();
}

Expand Down Expand Up @@ -133,6 +133,17 @@ public void removeDatabase(ViewerDatabase database, Path lobFolder) throws Viewe
public void addTable(ViewerTable table) throws ViewerException {
}

public void addRow(String databaseUUID, ViewerRow row) throws ViewerException {

RowsCollection collection = SolrRowsCollectionRegistry.get(databaseUUID);

try {
insertDocument(collection.getIndexName(), collection.toSolrDocument(row));
} catch (RequestNotValidException | GenericException | NotFoundException | AuthorizationDeniedException e) {
throw new ViewerException(e);
}
}

public void addRow(ViewerDatabaseFromToolkit viewerDatabase, ViewerRow row) throws ViewerException {

RowsCollection collection = SolrRowsCollectionRegistry.get(viewerDatabase.getUUID());
Expand All @@ -146,7 +157,8 @@ public void addRow(ViewerDatabaseFromToolkit viewerDatabase, ViewerRow row) thro

public <T extends IsIndexed> IndexResult<T> find(Class<T> classToReturn, Filter filter, Sorter sorter,
Sublist sublist, Facets facets) throws GenericException, RequestNotValidException {
return SolrUtils.find(client, SolrDefaultCollectionRegistry.get(classToReturn), filter, sorter, sublist, facets);
return SolrUtils.find(client, SolrDefaultCollectionRegistry.get(classToReturn), filter, sorter, sublist, facets,
new ArrayList<>());
}

public <T extends IsIndexed> Long count(Class<T> classToReturn, Filter filter)
Expand All @@ -164,10 +176,14 @@ public IndexResult<ViewerRow> findRows(String databaseUUID, Filter filter, Sorte
return SolrUtils.findRows(client, databaseUUID, filter, sorter, sublist, facets);
}

public InputStream findRowsCSV(String databaseUUID, Filter filter, Sorter sorter, Sublist sublist,
List<String> fields) throws GenericException, RequestNotValidException {
return SolrUtils.findCSV(client, SolrRowsCollectionRegistry.get(databaseUUID).getIndexName(), filter, sorter,
sublist, fields);
public IndexResult<ViewerRow> findRows(String databaseUUID, Filter filter, Sorter sorter, Sublist sublist,
Facets facets, List<String> fieldsToReturn) throws GenericException, RequestNotValidException {
return SolrUtils.findRows(client, databaseUUID, filter, sorter, sublist, facets, fieldsToReturn);
}

public IterableIndexResult findAllRows(String databaseUUID, final Filter filter, final Sorter sorter,
final List<String> fieldsToReturn) {
return new IterableIndexResult(client, databaseUUID, filter, sorter, fieldsToReturn);
}

public <T extends IsIndexed> Long countRows(String databaseUUID, Filter filter)
Expand Down Expand Up @@ -300,8 +316,8 @@ public void markDatabaseCollection(final String databaseUUID, ViewerDatabase.Sta
updateDatabaseFields(databaseUUID, Pair.of(ViewerConstants.SOLR_DATABASES_STATUS, status.toString()));
}

public void markDatabaseAsReady(ViewerDatabaseFromToolkit viewerDatabase) throws ViewerException {
updateDatabaseFields(viewerDatabase.getUUID(),
public void markDatabaseAsReady(final String databaseUUID) throws ViewerException {
updateDatabaseFields(databaseUUID,
Pair.of(ViewerConstants.SOLR_DATABASES_STATUS, ViewerDatabase.Status.AVAILABLE.toString()));
}

Expand Down
Loading

0 comments on commit f19c75c

Please sign in to comment.