Allows saving of files to filesystem (#312)

* Allows saving of files to filesystem
* Adds rolebacks and null checks to fileUpload
* Introduces apache.tika content type validation
* BREAKING: Refactors file validation

Co-authored-by: Daniel Koch <koch@terrestris.de>
Co-authored-by: André Henn <henn@terrestris.de>

pom.xml

@@ -112,6 +112,7 @@
 
     <!-- Utils -->
     <commons.io.version>2.8.0</commons.io.version>
+    <tika.core.version>1.26</tika.core.version>
     <reflections.version>0.9.12</reflections.version>
     <evo-inflector.version>1.2.2</evo-inflector.version>
 
@@ -419,6 +420,13 @@
         <version>${commons.io.version}</version>
       </dependency>
 
+      <!-- Apache Tika -->
+      <dependency>
+        <groupId>org.apache.tika</groupId>
+        <artifactId>tika-core</artifactId>
+        <version>${tika.core.version}</version>
+      </dependency>
+
       <!-- Jackson -->
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>

shogun-boot/src/main/resources/db/migration/V0.8.0__FilePath.sql

@@ -0,0 +1,2 @@
+ALTER TABLE shogun.files ADD COLUMN IF NOT EXISTS path text;
+ALTER TABLE shogun.imagefiles ADD COLUMN IF NOT EXISTS path text;

shogun-config/src/main/java/de/terrestris/shogun/properties/UploadProperties.java

@@ -20,7 +20,6 @@
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.NestedConfigurationProperty;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.stereotype.Component;
 
 @Data
 @Configuration
@@ -33,4 +32,8 @@ public class UploadProperties {
     @NestedConfigurationProperty
     private ImageFileUploadProperties image;
 
+    private String basePath;
+
+    private String maxSize;
+
 }

shogun-config/src/main/resources/application-base.yml

@@ -134,3 +134,5 @@ upload:
       - image/png
       - image/svg+xml
       - image/tiff
+  basePath: /data
+  maxSize: 500M

shogun-lib/pom.xml

@@ -241,6 +241,12 @@
       <groupId>org.reflections</groupId>
       <artifactId>reflections</artifactId>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.tika</groupId>
+      <artifactId>tika-core</artifactId>
+    </dependency>
+
   </dependencies>
 
 </project>

shogun-lib/src/main/java/de/terrestris/shogun/lib/controller/BaseFileController.java

@@ -21,6 +21,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.MessageSource;
 import org.springframework.context.i18n.LocaleContextHolder;
 import org.springframework.core.GenericTypeResolver;
@@ -38,6 +39,9 @@ public abstract class BaseFileController<T extends BaseFileService<?, S>, S exte
 
     protected final Logger LOG = LogManager.getLogger(getClass());
 
+    @Value("${upload.basePath}")
+    protected String uploadBasePath;
+
     @Autowired
     protected T service;
 
@@ -97,31 +101,27 @@ public ResponseEntity<?> findOne(@PathVariable("fileUuid") UUID fileUuid) {
 
             if (entity.isPresent()) {
                 S file = entity.get();
-
-                LOG.info("Successfully got file with UUID {}", fileUuid);
-
                 final HttpHeaders responseHeaders = new HttpHeaders();
                 responseHeaders.setContentType(MediaType.parseMediaType(file.getFileType()));
                 responseHeaders.setContentDisposition(ContentDisposition.parse(
                     String.format("inline; filename=\"%s\"", file.getFileName())));
+                LOG.trace("Successfully got file with UUID {}", fileUuid);
 
-                return new ResponseEntity<>(file.getFile(), responseHeaders, HttpStatus.OK);
-            } else {
-                LOG.error("Could not find entity of type {} with UUID {}",
-                    getGenericClassName(), fileUuid);
-
-                throw new ResponseStatusException(
-                    HttpStatus.NOT_FOUND,
-                    messageSource.getMessage(
-                        "BaseController.NOT_FOUND",
-                        null,
-                        LocaleContextHolder.getLocale()
-                    )
-                );
+                byte[] fileData = service.getFileData(file);
+                return new ResponseEntity<>(fileData, responseHeaders, HttpStatus.OK);
             }
+
+            LOG.error("Could not find entity of type {} with UUID {}", getGenericClassName(), fileUuid);
+            throw new ResponseStatusException(
+                HttpStatus.NOT_FOUND,
+                messageSource.getMessage(
+                    "BaseController.NOT_FOUND",
+                    null,
+                    LocaleContextHolder.getLocale()
+                )
+            );
         } catch (AccessDeniedException ade) {
-            LOG.info("Access to entity of type {} with UUID {} is denied",
-                getGenericClassName(), fileUuid);
+            LOG.info("Access to entity of type {} with UUID {} is denied", getGenericClassName(), fileUuid);
 
             throw new ResponseStatusException(
                 HttpStatus.NOT_FOUND,
@@ -158,7 +158,7 @@ public S add(MultipartFile uploadedFile) {
 
         try {
 
-            service.isValidType(uploadedFile.getContentType());
+            service.isValid(uploadedFile);
 
             S persistedFile = service.create(uploadedFile);
 
@@ -194,6 +194,49 @@ public S add(MultipartFile uploadedFile) {
         }
     }
 
+    @PostMapping(value = "/uploadToFileSystem", consumes = "multipart/form-data")
+    @ResponseStatus(HttpStatus.CREATED)
+    public S addToFileSystem(MultipartFile uploadedFile) {
+        LOG.debug("Requested to upload a multipart-file and to save it to the file system");
+
+        try {
+
+            service.isValid(uploadedFile);
+
+            S persistedFile = service.create(uploadedFile, true);
+
+            LOG.info("Successfully uploaded file " + persistedFile.getFileName());
+
+            return persistedFile;
+        } catch (AccessDeniedException ade) {
+            LOG.info("Uploading entity of type {} is denied", getGenericClassName());
+
+            throw new ResponseStatusException(
+                HttpStatus.NOT_FOUND,
+                messageSource.getMessage(
+                    "BaseController.NOT_FOUND",
+                    null,
+                    LocaleContextHolder.getLocale()
+                ),
+                ade
+            );
+        } catch (ResponseStatusException rse) {
+            throw rse;
+        } catch (Exception e) {
+            LOG.error("Could not upload the file: " + e.getMessage());
+            LOG.trace("Full stack trace: ", e);
+
+            throw new ResponseStatusException(
+                HttpStatus.INTERNAL_SERVER_ERROR,
+                messageSource.getMessage(
+                    "BaseController.INTERNAL_SERVER_ERROR",
+                    null,
+                    LocaleContextHolder.getLocale()
+                )
+            );
+        }
+    }
+
     @DeleteMapping("/{fileUuid}")
     @ResponseStatus(HttpStatus.NO_CONTENT)
     public void delete(@PathVariable("fileUuid") UUID fileUuid) {

shogun-lib/src/main/java/de/terrestris/shogun/lib/model/File.java

@@ -18,12 +18,8 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.UUID;
-import javax.persistence.Cacheable;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Inheritance;
-import javax.persistence.InheritanceType;
-import javax.persistence.Table;
+import javax.persistence.*;
+
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -67,4 +63,9 @@ public class File extends BaseEntity {
     @Column(length = Integer.MAX_VALUE)
     @Getter @Setter
     private byte[] file;
+
+    @JsonIgnore
+    @Column
+    @Getter @Setter
+    private String path;
 }

shogun-lib/src/main/java/de/terrestris/shogun/lib/service/BaseFileService.java

@@ -18,17 +18,95 @@
 
 import de.terrestris.shogun.lib.model.File;
 import de.terrestris.shogun.lib.repository.BaseFileRepository;
+import de.terrestris.shogun.properties.UploadProperties;
+import org.apache.commons.io.FileUtils;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tomcat.util.http.fileupload.InvalidFileNameException;
+import org.apache.tomcat.util.http.fileupload.impl.InvalidContentTypeException;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
 import org.springframework.security.access.prepost.PostAuthorize;
+import org.springframework.util.PatternMatchUtils;
+import org.springframework.web.multipart.MultipartFile;
 
-import java.util.Optional;
-import java.util.UUID;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.*;
 
 public abstract class BaseFileService<T extends BaseFileRepository<S, Long> & JpaSpecificationExecutor<S>, S extends File> extends BaseService<T, S> implements IBaseFileService<T, S> {
 
+    @Autowired
+    private UploadProperties uploadProperties;
+
     @PostAuthorize("hasRole('ROLE_ADMIN') or hasPermission(returnObject.orElse(null), 'READ')")
     public Optional<S> findOne(UUID fileUuid) {
         return repository.findByFileUuid(fileUuid);
     }
 
+    public abstract S create(MultipartFile uploadFile, Boolean writeToSystem) throws Exception;
+
+    public void isValid(MultipartFile file) throws Exception {
+        if (file == null) {
+            throw new Exception("Given file is null.");
+        } else if (file.isEmpty()) {
+            throw new Exception("Given file is empty.");
+        }
+        this.isValidFileName(file.getOriginalFilename());
+        this.isValidType(file.getContentType());
+        this.verifyContentType(file);
+    }
+
+    public void verifyContentType(MultipartFile file) throws IOException, TikaException {
+        String contentType = file.getContentType();
+        String name = file.getName();
+        Metadata metadata = new Metadata();
+        metadata.set(Metadata.RESOURCE_NAME_KEY, name);
+        TikaConfig tika = new TikaConfig();
+        MediaType mediaType = tika.getDetector().detect(TikaInputStream.get(file.getBytes()), metadata);
+        if (!mediaType.toString().equals(contentType)) {
+            throw new IOException("Mediatype validation failed. Passed content type is " + contentType + " but detected mediatype is " + mediaType);
+        }
+    }
+
+    public void isValidType(String contentType) throws InvalidContentTypeException {
+        List<String> supportedContentTypes = getSupportedContentTypes();
+        boolean isMatch = PatternMatchUtils.simpleMatch(supportedContentTypes.toArray(new String[supportedContentTypes.size()]), contentType);
+        if (!isMatch) {
+            throw new InvalidContentTypeException("Unsupported content type for upload!");
+        }
+    }
+
+    public void isValidFileName(String fileName) throws InvalidFileNameException {
+        List<String> illegalCharacters = Arrays.asList("\\", "/", ":", "*", "?", "\"", "<", ">", "|", "\\0", "\\n");
+        if (illegalCharacters.stream().anyMatch(fileName::contains)) {
+            throw new InvalidFileNameException(fileName, "Filename contains illegal chracters. [\\, /, :, *, ?, \", <, >, |, \\0, \\n]");
+        }
+    }
+
+    /**
+     * Get the file data as bytearray. Depends on storage strategy (DB vs. disk);
+     *
+     * @param file
+     * @return
+     * @throws IOException
+     */
+    public byte[] getFileData(S file) throws IOException {
+        if (file.getPath() == null) {
+            LOG.trace("… load file from database");
+            return file.getFile();
+        }
+        java.io.File dataFile = new java.io.File(uploadProperties.getBasePath() + "/" + file.getPath());
+        if (dataFile.exists()) {
+            LOG.trace("… load file from disk");
+            return FileUtils.readFileToByteArray(dataFile);
+        } else {
+            LOG.error("Could not load File {} from disk", file.getId());
+            throw new FileNotFoundException("Could not load File " + file.getId() + " from disk");
+        }
+    }
+
 }