From 635d74a8d1ffacc983ba53f876c1cd024898d8ad Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Fri, 4 Aug 2023 13:52:03 +0200 Subject: [PATCH] Ufal 325/be preview files from zip (#388) * add the controller, file for handling the logic, wrapper and testing * add Id for the object response * add controller for handling download feature * extract zip each time the preview section call api * handle the restriction for download * add redirect feature and handle large files upload * add license header for new files * format style code * change wrong spelling * add license header --------- Co-authored-by: HuynhKhoa1601 --- .../main/java/org/dspace/util/FileInfo.java | 40 ++ .../dspace/util/FileTreeViewGenerator.java | 89 ++++ .../app/rest/MetadataBitstreamController.java | 188 +++++++++ .../MetadataBitstreamWrapperConverter.java | 52 +++ .../model/MetadataBitstreamWrapperRest.java | 123 ++++++ .../MetadataBitstreamWrapperResource.java | 19 + .../wrapper/MetadataBitstreamWrapper.java | 74 ++++ .../MetadataBitstreamRestRepository.java | 395 ++++++++++++++++++ .../rest/MetadataBitstreamControllerIT.java | 131 ++++++ .../MetadataBitstreamRestRepositoryIT.java | 192 +++++++++ 10 files changed, 1303 insertions(+) create mode 100644 dspace-api/src/main/java/org/dspace/util/FileInfo.java create mode 100644 dspace-api/src/main/java/org/dspace/util/FileTreeViewGenerator.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataBitstreamController.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/MetadataBitstreamWrapperConverter.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataBitstreamWrapperRest.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/MetadataBitstreamWrapperResource.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/model/wrapper/MetadataBitstreamWrapper.java create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java create mode 100644 dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java diff --git a/dspace-api/src/main/java/org/dspace/util/FileInfo.java b/dspace-api/src/main/java/org/dspace/util/FileInfo.java new file mode 100644 index 00000000000..4bc33f79449 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/util/FileInfo.java @@ -0,0 +1,40 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.util; + +import java.util.Hashtable; +/** + * This class is used to store the information about a file or a directory + * + * @author longtv + */ +public class FileInfo { + + public String name; + public String content; + public String size; + public boolean isDirectory; + + public Hashtable sub = null; + + public FileInfo(String name) { + this.name = name; + sub = new Hashtable(); + isDirectory = true; + } + public FileInfo(String content, boolean isDirectory) { + this.content = content; + this.isDirectory = isDirectory; + } + + public FileInfo(String name, String size) { + this.name = name; + this.size = size; + isDirectory = false; + } +} \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/util/FileTreeViewGenerator.java b/dspace-api/src/main/java/org/dspace/util/FileTreeViewGenerator.java new file mode 100644 index 00000000000..4a74eb7b2b6 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/util/FileTreeViewGenerator.java @@ -0,0 +1,89 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.util; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +/** + * Generate a tree view of the file in a bitstream + * + * @author longtv + */ +public class FileTreeViewGenerator { + private FileTreeViewGenerator () { + } + + public static List parse(String data) throws ParserConfigurationException, IOException, SAXException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new InputSource(new StringReader(data))); + Element rootElement = document.getDocumentElement(); + NodeList nl = rootElement.getChildNodes(); + FileInfo root = new FileInfo("root"); + int limitFile = 100; + int countFile = 0; + Node n = nl.item(0); + do { + String fileInfo = n.getFirstChild().getTextContent(); + String f[] = fileInfo.split("\\|"); + String fileName = ""; + String path = f[0]; + long size = Long.parseLong(f[1]); + if (!path.endsWith("/")) { + fileName = path.substring(path.lastIndexOf('/') + 1); + if (path.lastIndexOf('/') != -1) { + path = path.substring(0, path.lastIndexOf('/')); + } else { + path = ""; + } + } + FileInfo current = root; + for (String p : path.split("/")) { + if (current.sub.containsKey(p)) { + current = current.sub.get(p); + } else { + FileInfo temp = new FileInfo(p); + current.sub.put(p, temp); + current = temp; + } + } + if (!fileName.isEmpty()) { + FileInfo temp = new FileInfo(fileName, humanReadableFileSize(size)); + current.sub.put(fileName, temp); + countFile++; + } + } while ((n = n.getNextSibling()) != null && countFile < limitFile); + return new ArrayList<>(root.sub.values()); + } + public static String humanReadableFileSize(long bytes) { + int thresh = 1024; + if (Math.abs(bytes) < thresh) { + return bytes + " B"; + } + String units[] = {"kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}; + int u = -1; + do { + bytes /= thresh; + ++u; + } while (Math.abs(bytes) >= thresh && u < units.length - 1); + return bytes + " " + units[u]; + } +} + diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataBitstreamController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataBitstreamController.java new file mode 100644 index 00000000000..f9d35517456 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/MetadataBitstreamController.java @@ -0,0 +1,188 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + + +import org.apache.commons.compress.archivers.zip.Zip64Mode; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.dspace.app.rest.exception.DSpaceBadRequestException; +import org.dspace.app.rest.model.MetadataBitstreamWrapper; +import org.dspace.app.rest.utils.ContextUtil; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.MissingLicenseAgreementException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.*; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Context; +import org.dspace.handle.service.HandleService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.dspace.core.Constants; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.Deflater; +import java.util.zip.ZipOutputStream; + +@RestController +@RequestMapping("/bitstream") +public class MetadataBitstreamController { + + @Autowired + BitstreamService bitstreamService; + + @Autowired + HandleService handleService; + @Autowired + private AuthorizeService authorizeService; + @GetMapping("/handle/{id}/{subId}/{fileName}") + public ResponseEntity downloadSingleFile( + @PathVariable("id") String id, + @PathVariable("subId") String subId, + @PathVariable("fileName") String fileName, + HttpServletRequest request, HttpServletResponse response) throws IOException { + String handleID = id + "/" + subId; + if (StringUtils.isBlank(handleID)) { + throw new DSpaceBadRequestException("handle cannot be null!"); + } + Context context = ContextUtil.obtainContext(request); + if (Objects.isNull(context)) { + throw new RuntimeException("Cannot obtain the context from the request."); + } + + DSpaceObject dso = null; + + try{ + dso = handleService.resolveToObject(context, handleID); + } catch (Exception e) { + throw new RuntimeException("Cannot resolve handle: " + handleID); + } + + if (dso != null && dso instanceof Item) { + Item item = (Item) dso; + List bundles = item.getBundles(); + for (Bundle bundle: + bundles) { + for (Bitstream bitstream: + bundle.getBitstreams()) { + try { + authorizeService.authorizeAction(context, bitstream, Constants.READ); + } catch (MissingLicenseAgreementException e) { + response.sendRedirect("http://localhost:4000/bitstream/" + bitstream.getID() + "/download"); + } catch (AuthorizeException e) { + response.sendRedirect("http://localhost:4000" + "/login"); + } catch (SQLException e) { + response.sendRedirect("http://localhost:4000" + "/login"); + } + String btName = bitstream.getName(); + if (btName.equalsIgnoreCase(fileName)) { + try { + BitstreamFormat bitstreamFormat = bitstream.getFormat(context); + if (bitstreamFormat == null || bitstreamFormat.getExtensions() == null || bitstreamFormat.getExtensions().size() == 0) { +// throw new RuntimeException("Cannot find the bitstream format."); + } + InputStream inputStream = bitstreamService.retrieve(context, bitstream); + InputStreamResource resource = new InputStreamResource(inputStream); + HttpHeaders header = new HttpHeaders(); + header.add(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=" + fileName); +// "attachment; filename=" + fileName +".pdf"); + header.add("Cache-Control", "no-cache, no-store, must-revalidate"); + header.add("Pragma", "no-cache"); + header.add("Expires", "0"); + return ResponseEntity.ok() + .headers(header) + .contentLength(inputStream.available()) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + } + } + + return null; + } + + + @GetMapping("/allzip") + public void downloadFileZip(@RequestParam("handleId") String handleId, + HttpServletResponse response, + HttpServletRequest request) throws IOException, SQLException, AuthorizeException { + //TODO handle authorization + if (StringUtils.isBlank(handleId)) { + throw new DSpaceBadRequestException("handle cannot be null!"); + } + Context context = ContextUtil.obtainContext(request); + if (Objects.isNull(context)) { + throw new RuntimeException("Cannot obtain the context from the request."); + } + + DSpaceObject dso = null; + String name = ""; + try{ + dso = handleService.resolveToObject(context, handleId); + } catch (Exception e) { + throw new RuntimeException("Cannot resolve handle: " + handleId); + } + + if (dso != null && dso instanceof Item) { + Item item = (Item) dso; + name = item.getName() + ".zip"; + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, String.format("attachment;filename=\"%s\"", name)); + response.setContentType("application/zip"); + List bundles = item.getBundles("ORIGINAL"); + + + ZipArchiveOutputStream zip = new ZipArchiveOutputStream(response.getOutputStream()); + zip.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.ALWAYS); + zip.setLevel(Deflater.NO_COMPRESSION); + for (Bundle original : bundles) { + List bss = original.getBitstreams(); + for (Bitstream bitstream : bss) { + try { + authorizeService.authorizeAction(context, bitstream, Constants.READ); + } catch (AuthorizeException e) { + response.sendRedirect("http://localhost:4000" + "/login"); + } catch (SQLException e) { + response.sendRedirect("http://localhost:4000" + "/login"); + } + String filename = bitstream.getName(); + ZipArchiveEntry ze = new ZipArchiveEntry(filename); + zip.putArchiveEntry(ze); + InputStream is = bitstreamService.retrieve(context, bitstream); + IOUtils.copy(is, zip); + zip.closeArchiveEntry(); + is.close(); + } + } + zip.close(); + response.getOutputStream().flush(); + } + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/MetadataBitstreamWrapperConverter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/MetadataBitstreamWrapperConverter.java new file mode 100644 index 00000000000..50007f3dfe6 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/MetadataBitstreamWrapperConverter.java @@ -0,0 +1,52 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.converter; + +import org.dspace.app.rest.model.MetadataBitstreamWrapper; +import org.dspace.app.rest.model.MetadataBitstreamWrapperRest; +import org.dspace.app.rest.model.MetadataValueWrapper; +import org.dspace.app.rest.model.MetadataValueWrapperRest; +import org.dspace.app.rest.projection.Projection; +import org.dspace.core.Context; +import org.dspace.util.FileTreeViewGenerator; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Component +public class MetadataBitstreamWrapperConverter implements DSpaceConverter { + + @Lazy + @Autowired + private ConverterService converter; + + + @Autowired + private BitstreamConverter bitstreamConverter; + + @Override + public MetadataBitstreamWrapperRest convert(MetadataBitstreamWrapper modelObject, Projection projection) { + MetadataBitstreamWrapperRest bitstreamWrapperRest = new MetadataBitstreamWrapperRest(); + bitstreamWrapperRest.setProjection(projection); + bitstreamWrapperRest.setName(modelObject.getBitstream().getName()); + bitstreamWrapperRest.setId(modelObject.getBitstream().getID().toString()); + bitstreamWrapperRest.setDescription(modelObject.getDescription()); + bitstreamWrapperRest.setChecksum(modelObject.getBitstream().getChecksum()); + bitstreamWrapperRest.setFileSize(FileTreeViewGenerator.humanReadableFileSize(modelObject.getBitstream().getSizeBytes())); + bitstreamWrapperRest.setFileInfo(modelObject.getFileInfo()); + bitstreamWrapperRest.setHref(modelObject.getHref()); + bitstreamWrapperRest.setFormat(modelObject.getFormat()); + bitstreamWrapperRest.setCanPreview(modelObject.isCanPreview()); + return bitstreamWrapperRest; + } + + @Override + public Class getModelClass() { + return MetadataBitstreamWrapper.class; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataBitstreamWrapperRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataBitstreamWrapperRest.java new file mode 100644 index 00000000000..e2a011c4f77 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/MetadataBitstreamWrapperRest.java @@ -0,0 +1,123 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.model; + +import org.dspace.app.rest.RestResourceController; +import org.dspace.util.FileInfo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +public class MetadataBitstreamWrapperRest extends BaseObjectRest{ + public static final String NAME = "metadatabitstream"; + public static final String CATEGORY = RestAddressableModel.CORE; + + private String name; + private String description; + private String fileSize; + private String checksum; + private List fileInfo; + private String format; + private String href; + private boolean canPreview; + + public MetadataBitstreamWrapperRest(String name, String description, String fileSize, String checksum, List fileInfo, String format, String href, boolean canPreview) { + this.name = name; + this.description = description; + this.fileSize = fileSize; + this.checksum = checksum; + this.fileInfo = fileInfo; + this.format = format; + this.href = href; + this.canPreview = canPreview; + } + + public void setCanPreview(boolean canPreview) { + this.canPreview = canPreview; + } + + public boolean isCanPreview() { + return canPreview; + } + + public MetadataBitstreamWrapperRest() { + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public List getFileInfo() { + return fileInfo; + } + + public void setFileInfo(List fileInfo) { + this.fileInfo = fileInfo; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getFileSize() { + return fileSize; + } + + public void setFileSize(String fileSize) { + this.fileSize = fileSize; + } + + public String getChecksum() { + return checksum; + } + + public void setChecksum(String checksum) { + this.checksum = checksum; + } + + @Override + public String getCategory() { + return CATEGORY; + } + + + @Override + public Class getController() { + return RestResourceController.class; + } + + @Override + public String getType() { + return NAME; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/MetadataBitstreamWrapperResource.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/MetadataBitstreamWrapperResource.java new file mode 100644 index 00000000000..1c09c9d95f5 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/hateoas/MetadataBitstreamWrapperResource.java @@ -0,0 +1,19 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.model.hateoas; + +import org.dspace.app.rest.model.MetadataBitstreamWrapperRest; +import org.dspace.app.rest.model.hateoas.annotations.RelNameDSpaceResource; +import org.dspace.app.rest.utils.Utils; + +@RelNameDSpaceResource(MetadataBitstreamWrapperRest.NAME) +public class MetadataBitstreamWrapperResource extends DSpaceResource{ + public MetadataBitstreamWrapperResource(MetadataBitstreamWrapperRest data, Utils utils) { + super(data, utils); + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/wrapper/MetadataBitstreamWrapper.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/wrapper/MetadataBitstreamWrapper.java new file mode 100644 index 00000000000..0c8b684162f --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/wrapper/MetadataBitstreamWrapper.java @@ -0,0 +1,74 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.model; + +import org.dspace.content.Bitstream; +import org.dspace.util.FileInfo; + +import java.util.List; + +public class MetadataBitstreamWrapper { + private Bitstream bitstream; + private List fileInfo; + private String format; + private String description; + private String href; + + private boolean canPreview; + public MetadataBitstreamWrapper() { + } + + public MetadataBitstreamWrapper(Bitstream bitstream, List fileInfo, String format, String description, String href, boolean canPreview) { + this.bitstream = bitstream; + this.fileInfo = fileInfo; + this.format = format; + this.description = description; + this.href = href; + this.canPreview = canPreview; + } + + public String getDescription() { + return description; + } + + public boolean isCanPreview() { + return canPreview; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public Bitstream getBitstream() { + return bitstream; + } + + public void setBitstream(Bitstream bitstream) { + this.bitstream = bitstream; + } + + public List getFileInfo() { + return fileInfo; + } + + public void setFileInfo(List fileInfo) { + this.fileInfo = fileInfo; + } +} diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java new file mode 100644 index 00000000000..bee16f565b2 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/repository/MetadataBitstreamRestRepository.java @@ -0,0 +1,395 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest.repository; + + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.dspace.app.rest.Parameter; +import org.dspace.app.rest.SearchRestMethod; +import org.dspace.app.rest.converter.BitstreamConverter; +import org.dspace.app.rest.converter.MetadataBitstreamWrapperConverter; +import org.dspace.app.rest.exception.DSpaceBadRequestException; +import org.dspace.app.rest.model.MetadataBitstreamWrapper; +import org.dspace.app.rest.model.MetadataBitstreamWrapperRest; +import org.dspace.app.rest.model.MetadataValueWrapper; +import org.dspace.app.util.Util; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.content.*; +import org.dspace.content.clarin.ClarinLicense; +import org.dspace.content.clarin.ClarinLicenseResourceMapping; +import org.dspace.content.service.BitstreamService; +import org.dspace.content.service.ItemService; +import org.dspace.content.service.clarin.ClarinLicenseResourceMappingService; +import org.dspace.core.Constants; +import org.dspace.core.Context; +import org.dspace.handle.service.HandleService; +import org.dspace.util.FileInfo; +import org.dspace.util.FileTreeViewGenerator; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Component; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.servlet.http.HttpServletRequest; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.*; +import java.nio.file.*; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +@Component(MetadataBitstreamWrapperRest.CATEGORY + "." + MetadataBitstreamWrapperRest.NAME) +public class MetadataBitstreamRestRepository extends DSpaceRestRepository{ + private static Logger log = org.apache.logging.log4j.LogManager.getLogger(MetadataBitstreamRestRepository.class); + @Autowired + HandleService handleService; + + @Autowired + BitstreamConverter bitstreamConverter; + + @Autowired + MetadataBitstreamWrapperConverter metadataBitstreamWrapperConverter; + @Autowired + ItemService itemService; + @Autowired + ClarinLicenseResourceMappingService licenseService; + + @Autowired + AuthorizeService authorizeService; + + @Autowired + BitstreamService bitstreamService; + + @SearchRestMethod(name = "byHandle") + public Page findByHandle(@Parameter(value = "handle", required = true) String handle, + @Parameter(value = "fileGrpType", required = false) String fileGrpType, + Pageable pageable) + throws SQLException, ParserConfigurationException, IOException, SAXException, AuthorizeException { + if (StringUtils.isBlank(handle)) { + throw new DSpaceBadRequestException("handle cannot be null!"); + } + List metadataValueWrappers = new ArrayList<>(); + Context context = obtainContext(); + if (Objects.isNull(context)) { + throw new RuntimeException("Cannot obtain the context from the request."); + } + HttpServletRequest request = getRequestService().getCurrentRequest().getHttpServletRequest(); + String contextPath = request.getContextPath(); + List rs = new ArrayList<>(); + DSpaceObject dso = null; + + try{ + dso = handleService.resolveToObject(context, handle); + } catch (Exception e) { + throw new RuntimeException("Cannot resolve handle: " + handle); + } + + if ( dso instanceof Item) { + Item item = (Item) dso; + List fileGrpTypes = Arrays.asList(fileGrpType.split(",")); + List bundles = findEnabledBundles(fileGrpTypes, item); + + for (Bundle bundle : + bundles) { + List bitstreams = new ArrayList<>(); + String use = bundle.getName(); + if ("THUMBNAIL".equals(use)) + { + Thumbnail thumbnail = itemService.getThumbnail(context, item, false); + if(thumbnail != null) { + bitstreams.add(thumbnail.getThumb()); + } + } + else + { + bitstreams = bundle.getBitstreams(); + } + + for (Bitstream bitstream : + bitstreams) { + List clarinLicenseResourceMappings = licenseService.findByBitstreamUUID(context, bitstream.getID()); + boolean canPreview = false; + if ( clarinLicenseResourceMappings != null && clarinLicenseResourceMappings.size() > 0) { + ClarinLicenseResourceMapping licenseResourceMapping = clarinLicenseResourceMappings.get(0); + ClarinLicense clarinLicense = licenseResourceMapping.getLicense(); + canPreview = clarinLicense.getClarinLicenseLabels().stream().anyMatch(clarinLicenseLabel -> clarinLicenseLabel.getLabel().equals("PUB")); + } + String identifier = null; + if (item != null && item.getHandle() != null) + { + identifier = "handle/" + item.getHandle(); + } + else if (item != null) + { + identifier = "item/" + item.getID(); + } + else + { + identifier = "id/" + bitstream.getID(); + } + String url = contextPath + "/bitstream/"+identifier; + try + { + if (bitstream.getName() != null) + { + url += "/" + Util.encodeBitstreamName(bitstream.getName(), "UTF-8"); + } + } + catch (UnsupportedEncodingException uee) + { + log.error("UnsupportedEncodingException", uee); + } + + url += "?sequence="+bitstream.getSequenceID(); + + String isAllowed = "n"; + try { + if (authorizeService.authorizeActionBoolean(context, bitstream, Constants.READ)) { + isAllowed = "y"; + } + } catch (SQLException e) {/* Do nothing */} + + url += "&isAllowed=" + isAllowed; + if (canPreview) { + List metadataValues = bitstream.getMetadata(); + // Filter out all metadata values that are not local to the bitstream + // Uncomment this if we want to show metadata values that are local to the bitstream +// metadataValues = metadataValues.stream().filter(metadataValue -> +// match("local", "bitstream", "file", metadataValue.getMetadataField())) +// .collect(Collectors.toList()); + List fileInfos = new ArrayList<>(); + InputStream inputStream = bitstreamService.retrieve(context, bitstream); + if (bitstream.getFormat(context).getExtensions().contains("zip")) { + String data = extractFile(inputStream, bitstream.getName().substring(0, bitstream.getName().lastIndexOf("."))); + fileInfos = FileTreeViewGenerator.parse(data); + } else { + if (bitstream.getFormat(context).getMIMEType().equals("text/plain")) { + String data = getFileContent(inputStream); + fileInfos.add(new FileInfo(data, false)); + } + } +// if (bitstream.getFormat(context).getMIMEType().equals("text/plain")) { +// List finalFileInfos = fileInfos; +// metadataValues.stream().map(MetadataValue::getValue).reduce((s, s2) -> s + s2).ifPresent(s -> finalFileInfos.add(new FileInfo(s, false))); +// fileInfos = finalFileInfos; +// } else { +// StringBuilder sb = new StringBuilder(); +// sb.append(""); +// for (MetadataValue metadataValue : +// metadataValues) { +// sb.append(""); +// sb.append(metadataValue.getValue()); +// sb.append(""); +// } +// sb.append(""); +// try { +// fileInfos = FileTreeViewGenerator.parse(sb.toString()); +// } catch (Exception e) { +// log.error(e.getMessage(), e); +// fileInfos = null; +// } +// } + MetadataBitstreamWrapper bts = new MetadataBitstreamWrapper(bitstream, fileInfos, bitstream.getFormat(context).getMIMEType(), bitstream.getFormatDescription(context), url, canPreview); + metadataValueWrappers.add(bts); + rs.add(metadataBitstreamWrapperConverter.convert(bts, utils.obtainProjection())); + } else { + MetadataBitstreamWrapper bts = new MetadataBitstreamWrapper(bitstream, new ArrayList<>(), bitstream.getFormat(context).getMIMEType(), bitstream.getFormatDescription(context), url, canPreview); + metadataValueWrappers.add(bts); + rs.add(metadataBitstreamWrapperConverter.convert(bts, utils.obtainProjection())); + continue; + } + } + } + } + + return new PageImpl<>(rs, pageable, rs.size()); + } + + protected List findEnabledBundles(List fileGrpTypes, Item item) throws SQLException + { + // Check if the user is requested a specific bundle or + // the all bundles. + List bundles; + if (fileGrpTypes.size() == 0) + { + bundles = item.getBundles(); + } + else + { + bundles = new ArrayList(); + for (String fileGrpType : fileGrpTypes) + { + for (Bundle newBundle : item.getBundles(fileGrpType)) + { + bundles.add(newBundle); + } + } + } + + return bundles; + } + + + private boolean match(String schema, String element, String qualifier, MetadataField field) + { + if (!element.equals(Item.ANY) && !element.equals(field.getElement())) + { + return false; + } + + if (qualifier == null) + { + if (field.getQualifier() != null) + { + return false; + } + } + else if (!qualifier.equals(Item.ANY)) + { + if (!qualifier.equals(field.getQualifier())) + { + return false; + } + } + + if (!schema.equals(Item.ANY)) + { + if (field.getMetadataSchema() != null && !field.getMetadataSchema().getName().equals(schema)) + { + return false; + } + } + return true; + } + + + public String extractFile(InputStream inputStream, String folderRootName) { + List filePaths = new ArrayList<>(); + Path tempFile = null; + FileSystem zipFileSystem = null; + + try { + tempFile = Files.createTempFile("temp", ".zip"); + Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); + + zipFileSystem = FileSystems.newFileSystem(tempFile, (ClassLoader) null); + Path root = zipFileSystem.getPath("/"); + Files.walk(root) + .forEach(path -> { + try { + long fileSize = Files.size(path); + if (Files.isDirectory(path)) { + filePaths.add(path.toString().substring(1) + "/|" + fileSize ); + } else { + filePaths.add(path.toString().substring(1) + "|" + fileSize ); + } + } catch (IOException e) { + e.printStackTrace(); + } + }); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (zipFileSystem != null) { + try { + zipFileSystem.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + if (tempFile != null) { + try { + Files.delete(tempFile); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + StringBuilder sb = new StringBuilder(); + sb.append(("")); + int count = 0; + List allFiles = filePaths; + for (String filePath : allFiles) { + if (!filePath.isEmpty() && filePath.length() > 3) { + if (filePath.contains(".")) { + count ++; + } + sb.append(""); + sb.append(filePath); + sb.append(""); + + if (count > 10) { + sb.append(""); + sb.append("/|0"); + sb.append(""); + sb.append(""); + sb.append("...too many files...|0"); + sb.append(""); + break; + } + } + } + sb.append(("")); + return sb.toString(); + } + + private static long calculateUncompressedSize(InputStream inputStream) throws IOException { + byte[] buffer = new byte[4096]; + long uncompressedSize = 0; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + uncompressedSize += bytesRead; + } + return uncompressedSize; + } + + public String getFileContent(InputStream inputStream) throws IOException { + StringBuilder content = new StringBuilder(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); + + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append("\n"); + } + + reader.close(); + return content.toString(); + } + + @Override + public MetadataBitstreamWrapperRest findOne(Context context, Integer integer) { + return null; + } + + @Override + public Page findAll(Context context, Pageable pageable) { + return null; + } + + @Override + public Class getDomainClass() { + return MetadataBitstreamWrapperRest.class; + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java new file mode 100644 index 00000000000..bad69cbb33a --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamControllerIT.java @@ -0,0 +1,131 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.io.IOUtils; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.authorize.AuthorizeException; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.builder.BitstreamBuilder; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.service.BitstreamService; +import org.dspace.core.Constants; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.result.ContentResultMatchers; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.sql.SQLException; +import java.util.List; +import java.util.zip.Deflater; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +public class MetadataBitstreamControllerIT extends AbstractControllerIntegrationTest{ + private static final String METADATABITSTREAM_ENDPOINT = "/bitstream/"; + private static final String METADATABITSTREAM_DOWNLOAD_SINGLE_ENDPOINT = METADATABITSTREAM_ENDPOINT + "/handle"; + private static final String METADATABITSTREAM_DOWNLOAD_ALL_ENDPOINT = METADATABITSTREAM_ENDPOINT + "/allzip"; + private static final String AUTHOR = "Test author name"; + private Collection col; + + private Item publicItem; + private Bitstream bts; + + @Autowired + AuthorizeService authorizeService; + + @Autowired + BitstreamService bitstreamService; + + + @Override + public void setUp() throws Exception { + super.setUp(); + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + + col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection").build(); + + publicItem = ItemBuilder.createItem(context, col) + .withAuthor(AUTHOR) + .build(); + + String bitstreamContent = "ThisIsSomeDummyText"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + bts = BitstreamBuilder. + createBitstream(context, publicItem, is) + .withName("Bitstream") + .withDescription("Description") + .withMimeType("application/zip") + .build(); + } + context.restoreAuthSystemState(); + } + + @Test + public void downloadSingleFileNullPathVariable() throws Exception { + getClient().perform(get(METADATABITSTREAM_DOWNLOAD_SINGLE_ENDPOINT)).andExpect(status().isNotFound()); + } + + @Test + public void downloadSingleFileWithAuthorize() throws Exception { + InputStream ip = bitstreamService.retrieve(context, bts); + String token = getAuthToken(admin.getEmail(), password); + getClient(token).perform(get(METADATABITSTREAM_DOWNLOAD_SINGLE_ENDPOINT + "/" + publicItem.getHandle() + "/" + bts.getName())) + .andExpect(status().isOk()) + .andExpect(content().contentType("application/octet-stream;charset=UTF-8")) + .andExpect(content().bytes(IOUtils.toByteArray(ip))); + } + + @Test + public void downloadSingleFileWithNoAuthorize() throws Exception { + getClient().perform(get(METADATABITSTREAM_DOWNLOAD_SINGLE_ENDPOINT + "/" + publicItem.getHandle() + "/" + bts.getName())) + .andExpect(status().is3xxRedirection()); + } + + @Test + public void downloadAllZip() throws Exception { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ZipArchiveOutputStream zip = new ZipArchiveOutputStream(byteArrayOutputStream); + zip.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.ALWAYS); + zip.setLevel(Deflater.NO_COMPRESSION); + ZipArchiveEntry ze = new ZipArchiveEntry(bts.getName()); + zip.putArchiveEntry(ze); + InputStream is = bitstreamService.retrieve(context, bts); + org.apache.commons.compress.utils.IOUtils.copy(is, zip); + zip.closeArchiveEntry(); + is.close(); + zip.close(); + + String token = getAuthToken(admin.getEmail(), password); + getClient(token).perform(get(METADATABITSTREAM_DOWNLOAD_ALL_ENDPOINT ).param("handleId", publicItem.getHandle())) + .andExpect(status().isOk()) + .andExpect(content().bytes(byteArrayOutputStream.toByteArray())); + + } + + +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java new file mode 100644 index 00000000000..1f02fca2d72 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/MetadataBitstreamRestRepositoryIT.java @@ -0,0 +1,192 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import org.apache.commons.codec.CharEncoding; +import org.apache.commons.io.IOUtils; +import org.dspace.app.rest.test.AbstractControllerIntegrationTest; +import org.dspace.app.util.Util; +import org.dspace.authorize.service.AuthorizeService; +import org.dspace.builder.*; +import org.dspace.content.*; +import org.dspace.content.clarin.ClarinLicense; +import org.dspace.content.clarin.ClarinLicenseResourceMapping; +import org.dspace.content.service.clarin.ClarinLicenseResourceMappingService; +import org.dspace.core.Constants; +import org.dspace.util.FileTreeViewGenerator; +import org.hamcrest.Matchers; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.sql.SQLException; +import java.util.List; + +import static org.hamcrest.Matchers.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; + +public class MetadataBitstreamRestRepositoryIT extends AbstractControllerIntegrationTest { + + private static final String HANDLE_ID = "123456789/36"; + private static final String METADATABITSTREAM_ENDPOINT = "/api/core/metadatabitstream/"; + private static final String METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT = METADATABITSTREAM_ENDPOINT + "search/byHandle"; + private static final String FILE_GRP_TYPE = "ORIGINAL"; + private static final String AUTHOR = "Test author name"; + private Collection col; + + private Item publicItem; + private Bitstream bts; + private Bundle bundle; + private Boolean canPreview = false; + + @Autowired + ClarinLicenseResourceMappingService licenseService; + + @Autowired + AuthorizeService authorizeService; + + + @Test + public void findByHandleNullHandle() throws Exception { + getClient().perform(get(METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT)) + .andExpect(status().isBadRequest()); + } + + @Test + public void findByHandle() throws Exception { + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + + col = CollectionBuilder.createCollection(context, parentCommunity).withName("Collection").build(); + + publicItem = ItemBuilder.createItem(context, col) + .withAuthor(AUTHOR) + .build(); + + String bitstreamContent = "ThisIsSomeDummyText"; + try (InputStream is = IOUtils.toInputStream(bitstreamContent, CharEncoding.UTF_8)) { + bts = BitstreamBuilder. + createBitstream(context, publicItem, is) + .withName("Bitstream") + .withDescription("Description") + .withMimeType("application/x-gzip") + .build(); + } + + String identifier = null; + if (publicItem != null && publicItem.getHandle() != null) + { + identifier = "handle/" + publicItem.getHandle(); + } + else if (publicItem != null) + { + identifier = "item/" + publicItem.getID(); + } + else + { + identifier = "id/" + bts.getID(); + } + String url = "/bitstream/"+identifier+"/"; + try + { + if (bts.getName() != null) + { + url += Util.encodeBitstreamName(bts.getName(), "UTF-8"); + } + } + catch (UnsupportedEncodingException uee) + { + + } + + url += "?sequence=" + bts.getSequenceID(); + + String isAllowed = "n"; + try { + if (authorizeService.authorizeActionBoolean(context, bts, Constants.READ)) { + isAllowed = "y"; + } + } catch (SQLException e) {/* Do nothing */} + + url += "&isAllowed=" + isAllowed; + + context.restoreAuthSystemState(); + List bundles = publicItem.getBundles(FILE_GRP_TYPE); + for (Bundle bundle : bundles) { + bundle.getBitstreams().stream().forEach(bitstream -> { + List clarinLicenseResourceMappings = null; + try { + clarinLicenseResourceMappings = licenseService.findByBitstreamUUID(context, bitstream.getID()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + if ( clarinLicenseResourceMappings != null && clarinLicenseResourceMappings.size() > 0) { + ClarinLicenseResourceMapping licenseResourceMapping = clarinLicenseResourceMappings.get(0); + ClarinLicense clarinLicense = licenseResourceMapping.getLicense(); + canPreview = clarinLicense.getClarinLicenseLabels().stream() + .anyMatch(clarinLicenseLabel -> clarinLicenseLabel.getLabel().equals("PUB")); + } + }); + } + getClient().perform(get(METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT) + .param("handle", publicItem.getHandle()) + .param("fileGrpType", FILE_GRP_TYPE)) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._embedded.metadatabitstreams").exists()) + .andExpect(jsonPath("$._embedded.metadatabitstreams").isArray()) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].name") + .value(Matchers.containsInAnyOrder(Matchers.containsString("Bitstream")))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].description") + .value(Matchers.containsInAnyOrder(Matchers.containsString(bts.getFormatDescription(context))))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].format") + .value(Matchers.containsInAnyOrder(Matchers.containsString(bts.getFormat(context).getMIMEType())))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].fileSize") + .value(Matchers.containsInAnyOrder(Matchers.containsString(FileTreeViewGenerator.humanReadableFileSize(bts.getSizeBytes()))))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].canPreview") + .value(Matchers.containsInAnyOrder(Matchers.is(canPreview)))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].fileInfo").exists()) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].checksum") + .value(Matchers.containsInAnyOrder(Matchers.containsString(bts.getChecksum())))) + .andExpect(jsonPath("$._embedded.metadatabitstreams[*].href") + .value(Matchers.containsInAnyOrder(Matchers.containsString(url)))); + + + } + + @Test + public void findByHandleEmptyFileGrpType() throws Exception { + getClient().perform(get(METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT) + .param("handle", HANDLE_ID) + .param("fileGrpType", "")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.page.totalElements", is(0))) + .andExpect(jsonPath("$.page.totalPages", is(0))) + .andExpect(jsonPath("$.page.size", is(20))) + .andExpect(jsonPath("$.page.number", is(0))) + .andExpect(jsonPath("$._links.self.href", Matchers.containsString(METADATABITSTREAM_SEARCH_BY_HANDLE_ENDPOINT + "?handle=" + HANDLE_ID + "&fileGrpType="))); + } + + @Test + public void searchMethodsExist() throws Exception { + + getClient().perform(get("/api/core/metadatabitstreams")) + .andExpect(status().is5xxServerError()); + + getClient().perform(get("/api/core/metadatabitstreams/search")) + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$._links.byHandle", notNullValue())); + } +}