From 09ac883bea6f1ce85c0b68ac78146fac7c85e30f Mon Sep 17 00:00:00 2001 From: Paul Boon Date: Mon, 18 Feb 2019 15:22:21 +0100 Subject: [PATCH] Cleanup, mostly formatting --- .gitignore | 4 +- .../source/installation/config.rst | 5 +- downloads/.gitignore | 1 - .../dataverse/CustomizationFilesServlet.java | 37 +- .../edu/harvard/iq/dataverse/DatasetPage.java | 21 +- .../edu/harvard/iq/dataverse/LoginPage.java | 2 +- .../iq/dataverse/api/BatchServiceBean.java | 79 ++- .../harvard/iq/dataverse/api/Datasets.java | 93 +-- .../filesystem/FileRecordJobListener.java | 72 +-- .../dataaccess/ImageThumbConverter.java | 196 +++--- .../iq/dataverse/export/ExportService.java | 70 +-- .../ingest/IngestableDataChecker.java | 94 +-- .../harvard/iq/dataverse/util/FileUtil.java | 584 +++++++++--------- .../iq/dataverse/util/SystemConfig.java | 2 - 14 files changed, 601 insertions(+), 659 deletions(-) diff --git a/.gitignore b/.gitignore index 5b56a680b5b..e1915ef7ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,6 @@ michael-local GPATH GTAGS GRTAGS -.idea -*.iml # OS generated files # ###################### .DS_Store @@ -32,7 +30,7 @@ scripts/installer/dvinstall/ oauth-credentials.md /src/main/webapp/oauth2/newAccount.html -#scripts/api/setup-all.sh* +scripts/api/setup-all.sh* # ctags generated tag file tags diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index fbc8cec9801..61a16ecd95e 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -179,12 +179,12 @@ To configure Shibboleth see the :doc:`shibboleth` section and to configure OAuth The ``authenticationproviderrow`` database table controls which "authentication providers" are available within Dataverse. Out of the box, a single row with an id of "builtin" will be present. For each user in Dataverse, the ``authenticateduserlookup`` table will have a value under ``authenticationproviderid`` that matches this id. For example, the default "dataverseAdmin" user will have the value "builtin" under ``authenticationproviderid``. Why is this important? Users are tied to a specific authentication provider but conversion mechanisms are available to switch a user from one authentication provider to the other. As explained in the :doc:`/user/account` section of the User Guide, a graphical workflow is provided for end users to convert from the "builtin" authentication provider to a remote provider. Conversion from a remote authentication provider to the builtin provider can be performed by a sysadmin with access to the "admin" API. See the :doc:`/api/native-api` section of the API Guide for how to list users and authentication providers as JSON. -Enabling a second authentication provider will result in the Log In page showing additional providers for your users to choose from. By default, the Log In page will show the "builtin" provider, but you can adjust this via the ``:DefaultAuthProvider`` configuration option. +Enabling a second authentication provider will result in the Log In page showing additional providers for your users to choose from. By default, the Log In page will show the "builtin" provider, but you can adjust this via the ``:DefaultAuthProvider`` configuration option. "Remote only" mode should be considered experimental until https://github.com/IQSS/dataverse/issues/2974 is resolved. For now, "remote only" means: - Shibboleth or OAuth has been enabled. -- ``:AllowSignUp`` is set to "false" per the :doc:`config` section to prevent users from creating local accounts via the web interface. Please note that local accounts can also be created via API, and the way to prevent this is to block the ``builtin-users`` endpoint or scramble (or remove) the ``BuiltinUsers.KEY`` database setting per the :doc:`config` section. +- ``:AllowSignUp`` is set to "false" per the :doc:`config` section to prevent users from creating local accounts via the web interface. Please note that local accounts can also be created via API, and the way to prevent this is to block the ``builtin-users`` endpoint or scramble (or remove) the ``BuiltinUsers.KEY`` database setting per the :doc:`config` section. - The "builtin" authentication provider has been disabled. Note that disabling the builting auth provider means that the API endpoint for converting an account from a remote auth provider will not work. This is the main reason why https://github.com/IQSS/dataverse/issues/2974 is still open. Converting directly from one remote authentication provider to another (i.e. from GitHub to Google) is not supported. Conversion from remote is always to builtin. Then the user initiates a conversion from builtin to remote. Note that longer term, the plan is to permit multiple login options to the same Dataverse account per https://github.com/IQSS/dataverse/issues/3487 (so all this talk of conversion will be moot) but for now users can only use a single login option, as explained in the :doc:`/user/account` section of the User Guide. In short, "remote only" might work for you if you only plan to use a single remote authentication provider such that no conversion between remote authentication providers will be necessary. File Storage: Local Filesystem vs. Swift vs. S3 @@ -1268,7 +1268,6 @@ This sets the base name (without dot and extension), if not set it defaults to ' ``curl -X PUT -d domainstats http://localhost:8080/api/admin/settings/:PiwikAnalyticsTrackerFileName`` - :FileFixityChecksumAlgorithm ++++++++++++++++++++++++++++ diff --git a/downloads/.gitignore b/downloads/.gitignore index dfc09bf74d9..9eb4cbadeeb 100644 --- a/downloads/.gitignore +++ b/downloads/.gitignore @@ -2,4 +2,3 @@ glassfish-4.1.zip solr-7.3.0.tgz weld-osgi-bundle-2.2.10.Final-glassfish4.jar schemaSpy_5.0.0.jar -solr-4.6.0.tgz diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java b/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java index e411f8807ee..faaa247920e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/CustomizationFilesServlet.java @@ -19,7 +19,6 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; - import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import javax.ejb.EJB; import org.apache.commons.io.IOUtils; @@ -30,11 +29,11 @@ */ @WebServlet(name = "CustomizationFilesServlet", urlPatterns = {"/CustomizationFilesServlet"}) public class CustomizationFilesServlet extends HttpServlet { - + @EJB SettingsServiceBean settingsService; - - + + /** * Processes requests for both HTTP GET and POST * methods. @@ -45,8 +44,8 @@ public class CustomizationFilesServlet extends HttpServlet { * @throws IOException if an I/O error occurs */ protected void processRequest(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - response.setContentType("text/html;charset=UTF-8"); + throws ServletException, IOException { + response.setContentType("text/html;charset=UTF-8"); String customFileType = request.getParameter("customFileType"); String filePath = getFilePath(customFileType); @@ -64,7 +63,7 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re StringBuilder responseData = new StringBuilder(); try (PrintWriter out = response.getWriter()) { - + while ((line = in.readLine()) != null) { responseData.append(line); out.println(line); @@ -90,37 +89,37 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re } } - + private String getFilePath(String fileTypeParam){ String nonNullDefaultIfKeyNotFound = ""; - + if (fileTypeParam.equals(CustomizationConstants.fileTypeHomePage)) { - + // Homepage return settingsService.getValueForKey(SettingsServiceBean.Key.HomePageCustomizationFile, nonNullDefaultIfKeyNotFound); - + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeHeader)) { - + // Header return settingsService.getValueForKey(SettingsServiceBean.Key.HeaderCustomizationFile, nonNullDefaultIfKeyNotFound); } else if (fileTypeParam.equals(CustomizationConstants.fileTypeFooter)) { - + // Footer return settingsService.getValueForKey(SettingsServiceBean.Key.FooterCustomizationFile, nonNullDefaultIfKeyNotFound); - + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeStyle)) { - + // Style (css) return settingsService.getValueForKey(SettingsServiceBean.Key.StyleCustomizationFile, nonNullDefaultIfKeyNotFound); - + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeLogo)) { // Logo for installation - appears in header return settingsService.getValueForKey(SettingsServiceBean.Key.LogoCustomizationFile, nonNullDefaultIfKeyNotFound); } - + return ""; } @@ -136,7 +135,7 @@ private String getFilePath(String fileTypeParam){ */ @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { + throws ServletException, IOException { processRequest(request, response); } @@ -150,7 +149,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) */ @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { + throws ServletException, IOException { processRequest(request, response); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 75d2bd8bafb..599586f9e9c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -207,7 +207,7 @@ public enum DisplayMode { @Inject ThumbnailServiceWrapper thumbnailServiceWrapper; @Inject - SettingsWrapper settingsWrapper; + SettingsWrapper settingsWrapper; @Inject ProvPopupFragmentBean provPopupFragmentBean; @@ -1202,7 +1202,7 @@ private void updateDatasetFieldInputLevels() { Format: { DatasetFieldType.id : DatasetField } --------------------------------------------------------- */ // Initialize Map - Map mapDatasetFields = new HashMap<>(); + Map mapDatasetFields = new HashMap<>(); // Populate Map for (DatasetField dsf : workingVersion.getFlatDatasetFields()) { @@ -1560,8 +1560,7 @@ private String init(boolean initFull) { try { privateUrl = commandEngine.submit(new GetPrivateUrlCommand(dvRequestService.getDataverseRequest(), dataset)); if (privateUrl != null) { - JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.privateurl.infoMessageAuthor", Arrays - .asList(getPrivateUrlLink(privateUrl)))); + JH.addMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.privateurl.infoMessageAuthor", Arrays.asList(getPrivateUrlLink(privateUrl)))); } } catch (CommandException ex) { // No big deal. The user simply doesn't have access to create or delete a Private URL. @@ -1781,7 +1780,7 @@ private void resetVersionUI() { if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.authorIdValue)) { subField.getDatasetFieldValues().get(0).setValue(creatorOrcidId); } - if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.authorIdType)) { + if (subField.getDatasetFieldType().getName().equals(DatasetFieldConstant.authorIdType)) { DatasetFieldType authorIdTypeDatasetField = fieldService.findByName(DatasetFieldConstant.authorIdType); subField.setSingleControlledVocabularyValue(fieldService.findControlledVocabularyValueByDatasetFieldTypeAndStrValue(authorIdTypeDatasetField, "ORCID", true)); } @@ -2071,7 +2070,7 @@ private String releaseDataset(boolean minor) { new PublishDatasetCommand(dataset, dvRequestService.getDataverseRequest(), minor) ); dataset = result.getDataset(); - // Sucessfully executing PublishDatasetCommand does not guarantee that the dataset + // Sucessfully executing PublishDatasetCommand does not guarantee that the dataset // has been published. If a publishing workflow is configured, this may have sent the // dataset into a workflow limbo, potentially waiting for a third party system to complete // the process. So it may be premature to show the "success" message at this point. @@ -2714,7 +2713,7 @@ public String save() { if (editMode == EditMode.CREATE) { if ( selectedTemplate != null ) { if ( isSessionUserAuthenticated() ) { - cmd = new CreateNewDatasetCommand(dataset, dvRequestService.getDataverseRequest(), false, selectedTemplate); + cmd = new CreateNewDatasetCommand(dataset, dvRequestService.getDataverseRequest(), false, selectedTemplate); } else { JH.addMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("dataset.create.authenticatedUsersOnly")); return null; @@ -2965,7 +2964,7 @@ public boolean isLockedInProgress() { public boolean isDatasetLockedInWorkflow() { return (dataset != null) - ? dataset.isLockedFor(DatasetLock.Reason.Workflow) + ? dataset.isLockedFor(DatasetLock.Reason.Workflow) : false; } @@ -3464,7 +3463,7 @@ public boolean isDesignatedDatasetThumbnail (FileMetadata fileMetadata) { * Items for the "Designated this image as the Dataset thumbnail: */ - private FileMetadata fileMetadataSelectedForThumbnailPopup = null; + private FileMetadata fileMetadataSelectedForThumbnailPopup = null; public void setFileMetadataSelectedForThumbnailPopup(FileMetadata fm){ fileMetadataSelectedForThumbnailPopup = fm; @@ -3538,7 +3537,7 @@ public void saveAsDesignatedThumbnail() { * Items for the "Tags (Categories)" popup. * */ - private FileMetadata fileMetadataSelectedForTagsPopup = null; + private FileMetadata fileMetadataSelectedForTagsPopup = null; public void setFileMetadataSelectedForTagsPopup(){ @@ -3849,7 +3848,7 @@ private void removeUnusedFileTagsFromDataset() { * Items for the "Advanced (Ingest) Options" popup. * */ - private FileMetadata fileMetadataSelectedForIngestOptionsPopup = null; + private FileMetadata fileMetadataSelectedForIngestOptionsPopup = null; public void setFileMetadataSelectedForIngestOptionsPopup(FileMetadata fm){ fileMetadataSelectedForIngestOptionsPopup = fm; diff --git a/src/main/java/edu/harvard/iq/dataverse/LoginPage.java b/src/main/java/edu/harvard/iq/dataverse/LoginPage.java index f1bb6899111..f743c7c7c61 100644 --- a/src/main/java/edu/harvard/iq/dataverse/LoginPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/LoginPage.java @@ -119,7 +119,7 @@ public void init() { authProvider = authSvc.getAuthenticationProvider(systemConfig.getDefaultAuthProvider()); random = new Random(); } - + public List listCredentialsAuthenticationProviders() { List infos = new LinkedList<>(); for ( String id : authSvc.getAuthenticationProviderIdsOfType( CredentialsAuthenticationProvider.class ) ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java index 7737a37adbd..2fbf45866c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java @@ -27,7 +27,7 @@ */ @Stateless public class BatchServiceBean { - private static final Logger logger = Logger.getLogger(BatchServiceBean.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(BatchServiceBean.class.getCanonicalName()); @EJB DataverseServiceBean dataverseService; @@ -41,43 +41,40 @@ public void processFilePath(String fileDir, String parentIdtf, DataverseRequest PrintWriter validationLog = null; PrintWriter cleanupLog = null; try { - JsonArrayBuilder status = Json.createArrayBuilder(); - Date timestamp = new Date(); - - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss"); + JsonArrayBuilder status = Json.createArrayBuilder(); + Date timestamp = new Date(); - validationLog = new PrintWriter(new FileWriter( "../logs/validationLog"+ formatter.format(timestamp)+".txt")); - cleanupLog = new PrintWriter(new FileWriter( "../logs/cleanupLog"+ formatter.format(timestamp)+".txt")); - File dir = new File(fileDir); - if (dir.isDirectory()) { - for (File file : dir.listFiles()) { - if (null != file) { - if (!file.isHidden()) { - if (file.isDirectory()) { - try { - status.add(handleDirectory(dataverseRequest, file, importType, validationLog, - cleanupLog, createDV)); - } catch (ImportException e) { - logger.log(Level.SEVERE, "Exception in handleDirectory() for " + file.getName(), e); - } - } else { - try { - status.add(importService.handleFile(dataverseRequest, owner, file, importType, - validationLog, cleanupLog)); - } catch (ImportException e) { - logger.log(Level.SEVERE, "Exception in handleFile() for " + file.getName(), e); - } + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss"); - } + validationLog = new PrintWriter(new FileWriter( "../logs/validationLog"+ formatter.format(timestamp)+".txt")); + cleanupLog = new PrintWriter(new FileWriter( "../logs/cleanupLog"+ formatter.format(timestamp)+".txt")); + File dir = new File(fileDir); + if (dir.isDirectory()) { + for (File file : dir.listFiles()) { + if (!file.isHidden()) { + if (file.isDirectory()) { + try { + status.add(handleDirectory(dataverseRequest, file, importType, validationLog, cleanupLog, createDV)); + } catch (ImportException e) { + logger.log(Level.SEVERE, "Exception in handleDirectory() for "+ file.getName(),e); } + } else { + try { + status.add(importService.handleFile(dataverseRequest, owner, file, importType, validationLog, cleanupLog)); + } catch(ImportException e) { + logger.log(Level.SEVERE, "Exception in handleFile() for "+ file.getName(),e); + } + } } - } else { - status.add(importService.handleFile(dataverseRequest, owner, dir, importType, validationLog, cleanupLog)); - } - } catch (Exception e) { - logger.log(Level.SEVERE, "Exception in processFilePath()", e); + } else { + status.add(importService.handleFile(dataverseRequest, owner, dir, importType, validationLog, cleanupLog)); + + } + } + catch(Exception e) { + logger.log(Level.SEVERE, "Exception in processFilePath()", e); } finally { validationLog.close(); cleanupLog.close(); @@ -98,20 +95,16 @@ public JsonArrayBuilder handleDirectory(DataverseRequest dataverseRequest, File } } for (File file : dir.listFiles()) { - if (null != file) { - if (!file.isHidden()) { - try { - JsonObjectBuilder fileStatus = importService.handleFile(dataverseRequest, owner, file, - importType, validationLog, cleanupLog); - status.add(fileStatus); - } catch (ImportException | IOException e) { - status.add(Json.createObjectBuilder().add("importStatus", - "Exception importing " + file.getName() + ", message = " + e.getMessage())); - } + if (!file.isHidden()) { + try { + JsonObjectBuilder fileStatus = importService.handleFile(dataverseRequest, owner, file, importType, validationLog, cleanupLog); + status.add(fileStatus); + } catch (ImportException | IOException e) { + status.add(Json.createObjectBuilder().add("importStatus", "Exception importing " + file.getName() + ", message = " + e.getMessage())); } } } return status; } -} \ No newline at end of file +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index a7e502dd491..4f868d90ae7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -125,8 +125,8 @@ public class Datasets extends AbstractApiBean { private static final Logger logger = Logger.getLogger(Datasets.class.getCanonicalName()); - - @Inject DataverseSession session; + + @Inject DataverseSession session; @EJB DatasetServiceBean datasetService; @@ -160,7 +160,7 @@ public class Datasets extends AbstractApiBean { @EJB EjbDataverseEngine commandEngine; - + @EJB IndexServiceBean indexService; @@ -282,7 +282,7 @@ public Response useDefaultCitationDate( @PathParam("id") String id) { @GET @Path("{id}/versions") public Response listVersions( @PathParam("id") String id ) { - return allowCors(response( req -> + return allowCors(response( req -> ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id)) ) .stream() .map( d -> json(d) ) @@ -293,7 +293,7 @@ public Response listVersions( @PathParam("id") String id ) { @Path("{id}/versions/{versionId}") public Response getVersion( @PathParam("id") String datasetId, @PathParam("versionId") String versionId) { return allowCors(response( req -> { - DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId)); + DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId)); return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found") : ok(json(dsv)); })); @@ -899,7 +899,7 @@ public Response createAssignment(String userOrGroup, @PathParam("identifier") St @GET @Path("{identifier}/assignments") public Response getAssignments(@PathParam("identifier") String id) { - return response( req -> + return response( req -> ok( execCommand( new ListRoleAssignments(req, findDatasetOrDie(id))) .stream().map(ra->json(ra)).collect(toJsonArray())) ); @@ -910,7 +910,7 @@ public Response getAssignments(@PathParam("identifier") String id) { public Response getPrivateUrlData(@PathParam("id") String idSupplied) { return response( req -> { PrivateUrl privateUrl = execCommand(new GetPrivateUrlCommand(req, findDatasetOrDie(idSupplied))); - return (privateUrl != null) ? ok(json(privateUrl)) + return (privateUrl != null) ? ok(json(privateUrl)) : error(Response.Status.NOT_FOUND, "Private URL not found."); }); } @@ -918,52 +918,11 @@ public Response getPrivateUrlData(@PathParam("id") String idSupplied) { @POST @Path("{id}/privateUrl") public Response createPrivateUrl(@PathParam("id") String idSupplied) { - return response( req -> + return response( req -> ok(json(execCommand( new CreatePrivateUrlCommand(req, findDatasetOrDie(idSupplied)))))); } - @POST - @Path("{id}/moveTo/{newDataverseAlias}") - public Response updateDatasetOwner(@PathParam("id") String idSupplied, @PathParam("newDataverseAlias") String newDataverseAliasSupplied) { - try { - User u = findUserOrDie(); - if (!u.isSuperuser()) { - return error(Response.Status.FORBIDDEN, "Not a superuser"); - } - final Dataset dataset = findDatasetOrDie(idSupplied); - - final Dataverse dataverse = dataverseService.findByAlias(newDataverseAliasSupplied); - if (dataverse == null) - return error( Response.Status.NOT_FOUND, "Dataverse alias '" + newDataverseAliasSupplied + "' not found" ); - - dataset.setOwner(dataverse); - final Dataset newDataset = datasetService.merge(dataset); - final Dataverse newDataverseOwner = newDataset.getOwner(); - final JsonObjectBuilder jsonbuilder = json(newDataset); - - return allowCors(ok(jsonbuilder.add("newDataverseOwner", (newDataverseOwner != null) ? json(newDataverseOwner) : null))); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - } - - @GET - @Path("{id}/owner") - public Response updateDatasetOwner(@PathParam("id") long idSupplied) { - final Dataset dataset = datasetService.find(idSupplied); - if (dataset == null) - return error( Response.Status.NOT_FOUND, "Dataset id '" + idSupplied + "' not found" ); - - final Dataverse dataverse = dataset.getOwner(); - final JsonObjectBuilder jsonbuilder = json(dataset); - - return allowCors(ok(jsonbuilder.add("DatasetOwner", (dataverse != null) ? json(dataverse) : null))); - - } - - - @DELETE @Path("{id}/privateUrl") public Response deletePrivateUrl(@PathParam("id") String idSupplied) { @@ -1260,13 +1219,13 @@ public Response returnToAuthor(@PathParam("id") String idSupplied, String jsonBo /** * Add a File to an existing Dataset - * + * * @param idSupplied * @param jsonData * @param fileInputStream * @param contentDispositionHeader * @param formDataBodyPart - * @return + * @return */ @POST @Path("{id}/add") @@ -1294,19 +1253,19 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, ); } - + // ------------------------------------- // (2) Get the Dataset Id - // + // // ------------------------------------- Dataset dataset; - + try { dataset = findDatasetOrDie(idSupplied); } catch (WrappedResponse wr) { - return wr.getResponse(); + return wr.getResponse(); } - + //------------------------------------ // (2a) Make sure dataset does not have package file // @@ -1326,8 +1285,8 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, // ------------------------------------- String newFilename = contentDispositionHeader.getFileName(); String newFileContentType = formDataBodyPart.getMediaType().toString(); - - + + // (2a) Load up optional params via JSON //--------------------------------------- OptionalFileParams optionalFileParams = null; @@ -1336,10 +1295,10 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, try { optionalFileParams = new OptionalFileParams(jsonData); } catch (DataFileTagException ex) { - return error( Response.Status.BAD_REQUEST, ex.getMessage()); + return error( Response.Status.BAD_REQUEST, ex.getMessage()); } - + //------------------- // (3) Create the AddReplaceFileHelper object //------------------- @@ -1387,11 +1346,11 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, } } - + } // end: addFileToDataset - + private void msg(String m){ //System.out.println(m); logger.fine(m); @@ -1402,8 +1361,8 @@ private void dashes(){ private void msgt(String m){ dashes(); msg(m); dashes(); } - - + + private T handleVersion( String versionId, DsVersionHandler hdl ) throws WrappedResponse { switch (versionId) { @@ -1426,7 +1385,7 @@ private T handleVersion( String versionId, DsVersionHandler hdl ) } } } - + private DatasetVersion getDatasetVersionOrDie( final DataverseRequest req, String versionNumber, final Dataset ds ) throws WrappedResponse { DatasetVersion dsv = execCommand( handleVersion(versionNumber, new DsVersionHandler>(){ @@ -1439,7 +1398,7 @@ public Command handleLatest() { public Command handleDraft() { return new GetDraftDatasetVersionCommand(req, ds); } - + @Override public Command handleSpecific(long major, long minor) { return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor); @@ -1455,7 +1414,7 @@ public Command handleLatestPublished() { } return dsv; } - + @GET @Path("{identifier}/locks") public Response getLocks(@PathParam("identifier") String id, @QueryParam("type") DatasetLock.Reason lockType) { diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java index be914d41130..4a6f48f1600 100644 --- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java +++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java @@ -78,11 +78,11 @@ @Named @Dependent public class FileRecordJobListener implements ItemReadListener, StepListener, JobListener { - + public static final String SEP = System.getProperty("file.separator"); - + private static final UserNotification.Type notifyType = UserNotification.Type.FILESYSTEMIMPORT; - + @Inject private JobContext jobContext; @@ -97,10 +97,10 @@ public class FileRecordJobListener implements ItemReadListener, StepListener, Jo @EJB ActionLogServiceBean actionLogServiceBean; - + @EJB DatasetServiceBean datasetServiceBean; - + @EJB DataFileServiceBean dataFileServiceBean; @@ -114,7 +114,7 @@ public class FileRecordJobListener implements ItemReadListener, StepListener, Jo @Inject @BatchProperty String checksumType; - + Properties jobParams; Dataset dataset; String mode; @@ -140,27 +140,27 @@ public void beforeRead() throws Exception { public void afterRead(Object item) throws Exception { // no-op } - + @Override public void onReadError(Exception ex) throws Exception { // no-op } - + @Override public void beforeJob() throws Exception { - Logger jobLogger; + Logger jobLogger; // initialize logger - // (the beforeJob() method gets executed before anything else; so we - // initialize the logger here. everywhere else will be retrieving - // it with Logger.getLogger(byname) - that should be giving us the + // (the beforeJob() method gets executed before anything else; so we + // initialize the logger here. everywhere else will be retrieving + // it with Logger.getLogger(byname) - that should be giving us the // same instance, created here - and not creating a new logger) jobLogger = LoggingUtil.getJobLogger(Long.toString(jobContext.getInstanceId())); // update job properties to be used elsewhere to determine dataset, user and mode JobOperator jobOperator = BatchRuntime.getJobOperator(); jobParams = jobOperator.getParameters(jobContext.getInstanceId()); - + // log job info jobLogger.log(Level.INFO, "Job ID = {0}", jobContext.getExecutionId()); jobLogger.log(Level.INFO, "Job Name = {0}", jobContext.getJobName()); @@ -219,7 +219,7 @@ public void afterJob() throws Exception { closeJobLoggerHandlers(); return; } - + // run reporting and notifications doReport(); @@ -235,11 +235,11 @@ public void afterJob() throws Exception { getJobLogger().log(Level.INFO, "Job start = " + step.getStartTime()); getJobLogger().log(Level.INFO, "Job end = " + step.getEndTime()); getJobLogger().log(Level.INFO, "Job exit status = " + step.getExitStatus()); - + closeJobLoggerHandlers(); } - + private void closeJobLoggerHandlers(){ // close the job logger handlers for (Handler h:getJobLogger().getHandlers()) { @@ -263,10 +263,10 @@ private boolean canRunJob() { return false; } - // if (!permissionServiceBean.userOn(user, dataset.getOwner()).has(Permission.EditDataset)) { - // getJobLogger().log(Level.SEVERE, "User doesn't have permission to import files into this dataset."); - // return false; - // } +// if (!permissionServiceBean.userOn(user, dataset.getOwner()).has(Permission.EditDataset)) { +// getJobLogger().log(Level.SEVERE, "User doesn't have permission to import files into this dataset."); +// return false; +// } if (dataset.getVersions().size() != 1) { getJobLogger().log(Level.SEVERE, "File system import is currently only supported for datasets with one version."); @@ -284,7 +284,7 @@ private boolean canRunJob() { * Generate all the job reports and user notifications. */ private void doReport() { - + try { String jobJson; @@ -315,7 +315,7 @@ private void doReport() { jobJson = new ObjectMapper().writeValueAsString(jobExecutionEntity); String logDir = System.getProperty("com.sun.aas.instanceRoot") + SEP + "logs" + SEP + "batch-jobs" + SEP; - + // [1] save json log to file LoggingUtil.saveJsonLog(jobJson, logDir, jobId); // [2] send user notifications - to all authors @@ -328,12 +328,12 @@ private void doReport() { List superUsers = authenticationServiceBean.findSuperUsers(); if (superUsers != null && !superUsers.isEmpty()) { superUsers.forEach((au) -> { - notificationServiceBean.sendNotification(au, timestamp, notifyType, datasetVersionId); + notificationServiceBean.sendNotification(au, timestamp, notifyType, datasetVersionId); }); } // [4] action log: store location of the full log to avoid truncation issues actionLogServiceBean.log(LoggingUtil.getActionLogRecord(user.getIdentifier(), jobExecution, - logDir + "job-" + jobId + ".log", jobId)); + logDir + "job-" + jobId + ".log", jobId)); } else { getJobLogger().log(Level.SEVERE, "Job execution is null"); @@ -350,9 +350,9 @@ private void doReport() { */ private String getDatasetGlobalId() { if (jobParams.containsKey("datasetId")) { - + String datasetId = jobParams.getProperty("datasetId"); - + dataset = datasetServiceBean.find(new Long(datasetId)); if (dataset != null) { @@ -412,33 +412,33 @@ private String getMode() { } /** - * Load the checksum manifest into an in memory HashMap, available to the step's read-process-write classes via the + * Load the checksum manifest into an in memory HashMap, available to the step's read-process-write classes via the * step context's transientUserData */ private void loadChecksumManifest() { - + // log job checksum type and how it was configured if (System.getProperty("checksumType") != null) { getJobLogger().log(Level.INFO, "Checksum type = " + System.getProperty("checksumType") + " ('checksumType' System property)"); } else { getJobLogger().log(Level.INFO, "Checksum type = " + checksumType + " (FileSystemImportJob.xml property)"); } - + // check system property first, otherwise use default property in FileSystemImportJob.xml String manifest; if (System.getProperty("checksumManifest") != null) { manifest = System.getProperty("checksumManifest"); - getJobLogger().log(Level.INFO, "Checksum manifest = " + manifest + " ('checksumManifest' System property)"); + getJobLogger().log(Level.INFO, "Checksum manifest = " + manifest + " ('checksumManifest' System property)"); } else { manifest = checksumManifest; getJobLogger().log(Level.INFO, "Checksum manifest = " + manifest + " (FileSystemImportJob.xml property)"); } // construct full path String manifestAbsolutePath = System.getProperty("dataverse.files.directory") - + SEP + dataset.getAuthority() - + SEP + dataset.getIdentifier() - + SEP + uploadFolder - + SEP + manifest; + + SEP + dataset.getAuthority() + + SEP + dataset.getIdentifier() + + SEP + uploadFolder + + SEP + manifest; getJobLogger().log(Level.INFO, "Reading checksum manifest: " + manifestAbsolutePath); Scanner scanner = null; try { @@ -460,9 +460,9 @@ private void loadChecksumManifest() { } } - + private Logger getJobLogger() { return Logger.getLogger("job-"+jobContext.getInstanceId()); } - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 95396292183..f273d215a3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -100,7 +100,7 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s return false; } - // similarly, if this is a harvested file: + // similarly, if this is a harvested file: if (file.isHarvested()) { logger.fine("thumbnails are not supported on harvested files at this time."); return false; @@ -124,14 +124,14 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s } - // Note that this method works on ALL file types for which thumbnail - // generation is supported - image/*, pdf, worldmap and geo-tagged tabular; - // not just on images! The type differentiation is handled inside - // isThumbnailAvailable(); if the thumbnail is not yet cached, that - // method will attempt to generate and cache it. And once it's cached, + // Note that this method works on ALL file types for which thumbnail + // generation is supported - image/*, pdf, worldmap and geo-tagged tabular; + // not just on images! The type differentiation is handled inside + // isThumbnailAvailable(); if the thumbnail is not yet cached, that + // method will attempt to generate and cache it. And once it's cached, // it is the same "auxiliary file", or an extra file with the .thumb[size] // extension - which is the same for all supported types. - // Note that this method is mainly used by the data access API methods. + // Note that this method is mainly used by the data access API methods. // Whenever a page needs a thumbnail, we prefer to rely on the Base64 // string version. public static InputStreamIO getImageThumbnailAsInputStream(StorageIO storageIO, int size) { @@ -140,8 +140,8 @@ public static InputStreamIO getImageThumbnailAsInputStream(StorageIO s return null; } - // If we got that far, it's now reasonable to expect that the thumbnail - // has been generated cached. + // If we got that far, it's now reasonable to expect that the thumbnail + // has been generated cached. InputStream cachedThumbnailInputStream = null; try { @@ -180,19 +180,19 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s return false; } - // We rely on ImageMagick to convert PDFs; so if it's not installed, - // better give up right away: + // We rely on ImageMagick to convert PDFs; so if it's not installed, + // better give up right away: if (!isImageMagickInstalled()) { return false; } File sourcePdfFile = null; - // We'll to get a local Path for this file - but if that is not available - // (i.e., if it's a file that's stored by a driver that does not provide - // direct file access - e.g., swift), we'll save this PDF in a temp file, - // will run the ImageMagick on it, and will save its output in another temp - // file, and will save it as an "auxiliary" file via the driver. + // We'll to get a local Path for this file - but if that is not available + // (i.e., if it's a file that's stored by a driver that does not provide + // direct file access - e.g., swift), we'll save this PDF in a temp file, + // will run the ImageMagick on it, and will save its output in another temp + // file, and will save it as an "auxiliary" file via the driver. boolean tempFilesRequired = false; try { @@ -243,11 +243,11 @@ private static boolean generatePDFThumbnail(StorageIO storageIO, int s return false; } - // If there was a local Path to the permanent location of the PDF file on the - // filesystem, the generatePDFThumbnailFromFile() method must have already saved - // the generated thumbnail as that Path with the .thumb* extension. But - // if this file is stored without a local Path, we'll have to save the - // generated thumbnail with via the storage driver: + // If there was a local Path to the permanent location of the PDF file on the + // filesystem, the generatePDFThumbnailFromFile() method must have already saved + // the generated thumbnail as that Path with the .thumb* extension. But + // if this file is stored without a local Path, we'll have to save the + // generated thumbnail with via the storage driver: if (tempFilesRequired) { try { logger.fine("attempting to save generated pdf thumbnail, as AUX file " + THUMBNAIL_SUFFIX + size); @@ -279,15 +279,15 @@ private static boolean generateImageThumbnail(StorageIO storageIO, int } /* - * Note that the "WorldMapThumbnail" generator does the exact same thing as the - * "regular image" thumbnail generator. - * The only difference is that the image generator uses the main file as - * as the source; and the one for the worldmap uses an auxiliary file - * with the ".img" extension (or the swift, etc. equivalent). This file is + * Note that the "WorldMapThumbnail" generator does the exact same thing as the + * "regular image" thumbnail generator. + * The only difference is that the image generator uses the main file as + * as the source; and the one for the worldmap uses an auxiliary file + * with the ".img" extension (or the swift, etc. equivalent). This file is * produced and dropped into the Dataset directory (Swift container, etc.) - * the first time the user actually runs WorldMap on the main file. + * the first time the user actually runs WorldMap on the main file. * Also note that it works the exact same way for tabular-mapped-as-worldmap - * files as well. + * files as well. */ private static boolean generateWorldMapThumbnail(StorageIO storageIO, int size) { @@ -323,8 +323,8 @@ private static boolean generateWorldMapThumbnail(StorageIO storageIO, } /* - * This is the actual workhorse method that does the rescaling of the full - * size image: + * This is the actual workhorse method that does the rescaling of the full + * size image: */ private static boolean generateImageThumbnailFromInputStream(StorageIO storageIO, int size, InputStream inputStream) { @@ -350,12 +350,12 @@ private static boolean generateImageThumbnailFromInputStream(StorageIO OutputStream outputStream = null; - // With some storage drivers, we can open a WritableChannel, or OutputStream - // to directly write the generated thumbnail that we want to cache; + // With some storage drivers, we can open a WritableChannel, or OutputStream + // to directly write the generated thumbnail that we want to cache; // Some drivers (like Swift) do not support that, and will give us an - // "operation not supported" exception. If that's the case, we'll have - // to save the output into a temp file, and then copy it over to the - // permanent storage using the DataAccess IO "save" command: + // "operation not supported" exception. If that's the case, we'll have + // to save the output into a temp file, and then copy it over to the + // permanent storage using the DataAccess IO "save" command: boolean tempFileRequired = false; File tempFile = null; @@ -382,7 +382,7 @@ private static boolean generateImageThumbnailFromInputStream(StorageIO rescaleImage(fullSizeImage, width, height, size, outputStream); /* - // while we are at it, let's make sure other size thumbnails are + // while we are at it, let's make sure other size thumbnails are // generated too: for (int s : (new int[]{DEFAULT_PREVIEW_SIZE, DEFAULT_THUMBNAIL_SIZE, DEFAULT_CARDIMAGE_SIZE})) { if (size != s && !thumbnailFileExists(fileLocation, s)) { @@ -452,20 +452,20 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) { return null; } - // skip the "isAvailable()" check - and just try to open the cached object. - // if we can't open it, then we'll try to generate it. In other words, we are doing it in - // the reverse order - and his way we can save one extra lookup, for a thumbnail + // skip the "isAvailable()" check - and just try to open the cached object. + // if we can't open it, then we'll try to generate it. In other words, we are doing it in + // the reverse order - and his way we can save one extra lookup, for a thumbnail // that's already cached - and on some storage media (specifically, S3) - // lookups are actually more expensive than reads. + // lookups are actually more expensive than reads. // (an experiment...) //if (!isThumbnailAvailable(storageIO, size)) { // logger.info("no thumbnail available for " + file.getStorageIdentifier()); // return null; //} - // we are skipping this StorageIO.open() call as well - since this + // we are skipping this StorageIO.open() call as well - since this // is another (potentially expensive) S3/swift lookup. - //storageIO.open(); - + //storageIO.open(); + Channel cachedThumbnailChannel = null; try { cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size); @@ -476,7 +476,7 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) { if (cachedThumbnailChannel == null) { logger.fine("Null channel for aux object " + THUMBNAIL_SUFFIX + size); - // try to generate, if not available: + // try to generate, if not available: boolean generated = false; if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) { generated = generateImageThumbnail(storageIO, size); @@ -487,7 +487,7 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) { } if (generated) { - // try to open again: + // try to open again: try { cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size); } catch (Exception ioEx) { @@ -517,10 +517,10 @@ private static String getImageAsBase64FromInputStream(InputStream inputStream) { int total = 0; // No, you don't want to try and inputStream.read() the entire thumbSize - // bytes at once; it's a thumbnail, but it can still be several K in size. - // And with some input streams - notably, with swift - you CANNOT read + // bytes at once; it's a thumbnail, but it can still be several K in size. + // And with some input streams - notably, with swift - you CANNOT read // more than 8192 bytes in one .read(). - + while ((bytes = inputStream.read(buffer)) > -1) { cachingByteStream.write(buffer, 0, bytes); total += bytes; @@ -553,8 +553,8 @@ private static String getImageAsBase64FromInputStream(InputStream inputStream) { * downloadable image via an API call. */ /* - * This is a version of the getImageAsBase64...() method that operates on - * a File; it's used for generating Dataverse and Dataset thumbnails + * This is a version of the getImageAsBase64...() method that operates on + * a File; it's used for generating Dataverse and Dataset thumbnails * from usr-uploaded images (i.e., from files not associated with datafiles) */ public static String getImageAsBase64FromFile(File imageFile) { @@ -582,11 +582,11 @@ public static String getImageAsBase64FromFile(File imageFile) { } /* - * This is a version of generateImageThumbnail...() that works directly on - * local files, for input and output. We still need it for various places - * in the application - when we process uploaded images that are not - * datafiles, etc. - * + * This is a version of generateImageThumbnail...() that works directly on + * local files, for input and output. We still need it for various places + * in the application - when we process uploaded images that are not + * datafiles, etc. + * */ public static String generateImageThumbnailFromFile(String fileLocation, int size) { @@ -638,10 +638,10 @@ public static String generateImageThumbnailFromFile(String fileLocation, int siz } /* - * This is another public version of generateImageThumbnail...() that works directly on + * This is another public version of generateImageThumbnail...() that works directly on * local files, for input and output. This one returns the output as Base64. - * Used by the DatasetWidgetsPage, to rescale the uploaded dataset logo. - * + * Used by the DatasetWidgetsPage, to rescale the uploaded dataset logo. + * */ public static String generateImageThumbnailFromFileAsBase64(File file, int size) { String thumbnailFileLocation = generateImageThumbnailFromFile(file.getAbsolutePath(), size); @@ -656,8 +656,8 @@ public static String generateImageThumbnailFromFileAsBase64(File file, int size) } // Public version of the rescaleImage() method; it takes the location of the output - // file as a string argument. This method is used by external utilities for - // rescaling the non-datafile Dataverse and Dataset logos. + // file as a string argument. This method is used by external utilities for + // rescaling the non-datafile Dataverse and Dataset logos. public static String rescaleImage(BufferedImage fullSizeImage, int width, int height, int size, String fileLocation) { String outputLocation = fileLocation + "." + THUMBNAIL_SUFFIX + size; File outputFile = new File(outputLocation); @@ -700,31 +700,31 @@ private static void rescaleImage(BufferedImage fullSizeImage, int width, int hei logger.fine("thumbnail dimensions: " + thumbWidth + "x" + thumbHeight); // If we are willing to spend a few extra CPU cycles to generate - // better-looking thumbnails, we can the SCALE_SMOOTH flag. - // SCALE_FAST trades quality for speed. + // better-looking thumbnails, we can the SCALE_SMOOTH flag. + // SCALE_FAST trades quality for speed. //logger.fine("Start image rescaling ("+size+" pixels), SCALE_FAST used;"); Image thumbImage = fullSizeImage.getScaledInstance(thumbWidth, thumbHeight, java.awt.Image.SCALE_FAST); //logger.fine("Finished image rescaling."); // if transparency is defined, we should preserve it in the png: - /* + /* OK, turns out *nothing* special needs to be done in order to preserve the transparency; the transparency is already there, because ImageIO.read() creates a BufferedImage with the color type BufferedImage.TYPE_INT_ARGB; - all we need to do, is to create the output BufferedImage lowRes, - below, with this same color type. The transparency was getting lost + all we need to do, is to create the output BufferedImage lowRes, + below, with this same color type. The transparency was getting lost only because that BufferedImage was made with TYPE_INT_RGB, thus stripping the transparency off. - + BufferedImage bufferedImageForTransparency = new BufferedImage(thumbWidth, thumbgetHeight, BufferedImage.TYPE_INT_ARGB); Graphics2D g2 = bufferedImageForTransparency.createGraphics(); g2.drawImage(thumbImage, 0, 0, null); g2.dispose(); - + int color = bufferedImageForTransparency.getRGB(0, 0); - + logger.info("color we'll be using for transparency: "+color); - + thumbImage = makeColorTransparent(bufferedImageForTransparency, new Color(color)); */ ImageWriter writer = null; @@ -771,9 +771,9 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) // it it doesn't exist yet, let's attempt to generate it: long sizeLimit = getThumbnailSizeLimitPDF(); - /* - * sizeLimit set to -1 means that generation of thumbnails on the fly - * is disabled: + /* + * sizeLimit set to -1 means that generation of thumbnails on the fly + * is disabled: */ logger.fine("pdf size limit: " + sizeLimit); @@ -782,9 +782,9 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) return null; } - /* + /* * sizeLimit set to 0 means no limit - generate thumbnails on the fly - * for all files, regardless of size. + * for all files, regardless of size. */ if (sizeLimit > 0) { long fileSize = 0; @@ -792,7 +792,7 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) try { fileSize = new File(fileLocation).length(); } catch (Exception ex) { - // + // } if (fileSize == 0 || fileSize > sizeLimit) { @@ -815,29 +815,29 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) if (new File(imageMagickExec).exists()) { - // Based on the lessons recently learned in production: + // Based on the lessons recently learned in production: // - use "-thumbnail" instead of "-resize"; // - use "-flatten" // - use "-strip" // - (maybe?) use jpeg instead of png - ? // - // (what we observed in production - 3.6.3, June 2014 - was that + // (what we observed in production - 3.6.3, June 2014 - was that // for very large TIFF images, when processed wihout the options // above, thumbnails produced were still obscenely *huge*; for ex., - // for a 100MB 3000x4000 px. TIFF file, the 275 px. PNG thumbnail - // produced was 5MB! - which, for a page with multiple high-res - // images (Grad. School of Design dv) resulted in taking forever - // to load... JPG thumbnails, similarly produced, were smaller, but - // still unnecessarily large. I was never able to figure out what - // was going on (full-rez color profiles still stored in the - // resized version - ??), but the combination above takes care of - // it and brings down the thumbnail size to under 50K, were it + // for a 100MB 3000x4000 px. TIFF file, the 275 px. PNG thumbnail + // produced was 5MB! - which, for a page with multiple high-res + // images (Grad. School of Design dv) resulted in taking forever + // to load... JPG thumbnails, similarly produced, were smaller, but + // still unnecessarily large. I was never able to figure out what + // was going on (full-rez color profiles still stored in the + // resized version - ??), but the combination above takes care of + // it and brings down the thumbnail size to under 50K, were it // belongs. :) // -- L.A. June 2014 String previewFileLocation = null; - // check if the "preview size" image is already available - and - // if not, generate it. this 400 pixel image will be used to + // check if the "preview size" image is already available - and + // if not, generate it. this 400 pixel image will be used to // generate smaller-size thumbnails. previewFileLocation = fileLocation + ".thumb" + DEFAULT_PREVIEW_SIZE; @@ -862,7 +862,7 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) return thumbFileLocation; /* - An alternative way of handling it: + An alternative way of handling it: while we are at it, let's generate *all* the smaller thumbnail sizes: for (int s : (new int[]{DEFAULT_THUMBNAIL_SIZE, DEFAULT_CARDIMAGE_SIZE})) { String thisThumbLocation = fileLocation + ".thumb" + s; @@ -870,7 +870,7 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) thisThumbLocation = runImageMagick(imageMagickExec, previewFileLocation, thisThumbLocation, s, "png"); } } - + // return the location of the thumbnail for the requested size: if (new File(thumbFileLocation).exists()) { return thumbFileLocation; @@ -931,24 +931,24 @@ private static boolean isPdfFileOverSizeLimit(long fileSize) { private static boolean isFileOverSizeLimit(String fileType, long fileSize) { long sizeLimit = getThumbnailSizeLimit(fileType); - /* - * sizeLimit set to -1 means that generation of thumbnails on the fly - * is disabled: + /* + * sizeLimit set to -1 means that generation of thumbnails on the fly + * is disabled: */ if (sizeLimit < 0) { return true; } - /* + /* * sizeLimit set to 0 means no limit - generate thumbnails on the fly - * for all files, regardless of size. + * for all files, regardless of size. */ if (sizeLimit == 0) { return false; } if (fileSize == 0 || fileSize > sizeLimit) { - // this is a broken file of size 0, or + // this is a broken file of size 0, or // this file is too large - no thumbnail: return true; } @@ -1009,9 +1009,9 @@ private static String findImageMagickConvert() { /* The method below takes a BufferedImage, and makes the specified color - transparent. Turns out we don't really need to do this explicitly, since - the original transparency can easily be preserved. - + transparent. Turns out we don't really need to do this explicitly, since + the original transparency can easily be preserved. + private static Image makeColorTransparent(final BufferedImage im, final Color color) { final ImageFilter filter = new RGBImageFilter() { // the color we are looking for (white)... Alpha bits are set to opaque diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index 8b54a889bdf..59c3c20c5ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -87,25 +87,25 @@ public List< String[]> getExportersLabels() { } public InputStream getExport(Dataset dataset, String formatName) throws ExportException, IOException { - // first we will try to locate an already existing, cached export - // for this format: + // first we will try to locate an already existing, cached export + // for this format: InputStream exportInputStream = getCachedExportFormat(dataset, formatName); if (exportInputStream != null) { return exportInputStream; } - // if it doesn't exist, we'll try to run the export: + // if it doesn't exist, we'll try to run the export: exportFormat(dataset, formatName); - // and then try again: + // and then try again: exportInputStream = getCachedExportFormat(dataset, formatName); if (exportInputStream != null) { return exportInputStream; } - // if there is no cached export still - we have to give up and throw + // if there is no cached export still - we have to give up and throw // an exception! throw new ExportException("Failed to export the dataset as " + formatName); @@ -141,9 +141,9 @@ public String getExportAsString(Dataset dataset, String formatName) { } - // This method goes through all the Exporters and calls - // the "chacheExport()" method that will save the produced output - // in a file in the dataset directory, on each Exporter available. + // This method goes through all the Exporters and calls + // the "chacheExport()" method that will save the produced output + // in a file in the dataset directory, on each Exporter available. public void exportAllFormats(Dataset dataset) throws ExportException { try { clearAllCachedFormats(dataset); @@ -171,8 +171,8 @@ public void exportAllFormats(Dataset dataset) throws ExportException { } catch (ServiceConfigurationError serviceError) { throw new ExportException("Service configuration error during export. " + serviceError.getMessage()); } - // Finally, if we have been able to successfully export in all available - // formats, we'll increment the "last exported" time stamp: + // Finally, if we have been able to successfully export in all available + // formats, we'll increment the "last exported" time stamp: dataset.setLastExportTime(new Timestamp(new Date().getTime())); @@ -194,10 +194,10 @@ public void clearAllCachedFormats(Dataset dataset) throws IOException { } } - // This method finds the exporter for the format requested, + // This method finds the exporter for the format requested, // then produces the dataset metadata as a JsonObject, then calls - // the "cacheExport()" method that will save the produced output - // in a file in the dataset directory. + // the "cacheExport()" method that will save the produced output + // in a file in the dataset directory. public void exportFormat(Dataset dataset, String formatName) throws ExportException { try { Iterator exporters = loader.iterator(); @@ -237,7 +237,7 @@ public Exporter getExporter(String formatName) throws ExportException { throw new ExportException("No such Exporter: " + formatName); } - // This method runs the selected metadata exporter, caching the output + // This method runs the selected metadata exporter, caching the output // in a file in the dataset directory / container based on its DOI: private void cacheExport(DatasetVersion version, String format, JsonObject datasetAsJson, Exporter exporter) throws ExportException { boolean tempFileRequired = false; @@ -246,10 +246,10 @@ private void cacheExport(DatasetVersion version, String format, JsonObject datas Dataset dataset = version.getDataset(); StorageIO storageIO = null; try { - // With some storage drivers, we can open a WritableChannel, or OutputStream - // to directly write the generated metadata export that we want to cache; + // With some storage drivers, we can open a WritableChannel, or OutputStream + // to directly write the generated metadata export that we want to cache; // Some drivers (like Swift) do not support that, and will give us an - // "operation not supported" exception. If that's the case, we'll have + // "operation not supported" exception. If that's the case, we'll have // to save the output into a temp file, and then copy it over to the // permanent storage using the IO "save" command: try { @@ -276,7 +276,7 @@ private void cacheExport(DatasetVersion version, String format, JsonObject datas exporter.exportDataset(version, datasetAsJson, outputStream); outputStream.flush(); outputStream.close(); - + logger.fine("Saving path as aux for temp file in: " + Paths.get(tempFile.getAbsolutePath())); storageIO.savePathAsAux(Paths.get(tempFile.getAbsolutePath()), "export_" + format + ".cached"); boolean tempFileDeleted = tempFile.delete(); @@ -306,9 +306,9 @@ private void clearCachedExport(Dataset dataset, String format) throws IOExceptio } - // This method checks if the metadata has already been exported in this - // format and cached on disk. If it has, it'll open the file and retun - // the file input stream. If not, it'll return null. + // This method checks if the metadata has already been exported in this + // format and cached on disk. If it has, it'll open the file and retun + // the file input stream. If not, it'll return null. private InputStream getCachedExportFormat(Dataset dataset, String formatName) throws ExportException, IOException { StorageIO dataAccess = null; @@ -335,20 +335,20 @@ private InputStream getCachedExportFormat(Dataset dataset, String formatName) th *to be compatible with storage drivers other than local filesystem. *Files.exists() would need to be discarded. * -- L.A. 4.8 */ - // public Long getCachedExportSize(Dataset dataset, String formatName) { - // try { - // if (dataset.getFileSystemDirectory() != null) { - // Path cachedMetadataFilePath = Paths.get(dataset.getFileSystemDirectory().toString(), "export_" + formatName + ".cached"); - // if (Files.exists(cachedMetadataFilePath)) { - // return cachedMetadataFilePath.toFile().length(); - // } - // } - // } catch (Exception ioex) { - // // don't do anything - we'll just return null - // } - // - // return null; - // } +// public Long getCachedExportSize(Dataset dataset, String formatName) { +// try { +// if (dataset.getFileSystemDirectory() != null) { +// Path cachedMetadataFilePath = Paths.get(dataset.getFileSystemDirectory().toString(), "export_" + formatName + ".cached"); +// if (Files.exists(cachedMetadataFilePath)) { +// return cachedMetadataFilePath.toFile().length(); +// } +// } +// } catch (Exception ioex) { +// // don't do anything - we'll just return null +// } +// +// return null; +// } public Boolean isXMLFormat(String provider) { try { Iterator exporters = loader.iterator(); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java index f3df2cf3a81..a73fb710ead 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java @@ -33,17 +33,17 @@ import org.apache.commons.io.IOUtils; /** - * This is a virtually unchanged DVN v2-3 implementation by + * This is a virtually unchanged DVN v2-3 implementation by * @author Akio Sone * - * incorporated into 4.0 by + * incorporated into 4.0 by * @author Leonid Andreev - * + * */ public class IngestableDataChecker implements java.io.Serializable { /** - * + * */ // static fields @@ -82,11 +82,11 @@ public class IngestableDataChecker implements java.io.Serializable { stataReleaseNumber.put((byte) 111, "rel_7scnd"); stataReleaseNumber.put((byte) 113, "rel_8_or_9"); stataReleaseNumber.put((byte) 114, "rel_10"); - stataReleaseNumber.put((byte) 115, "rel_12"); - // 116 was an in-house experimental version that was never + stataReleaseNumber.put((byte) 115, "rel_12"); + // 116 was an in-house experimental version that was never // released. // STATA v.13 introduced a new format, 117. It's a completely - // new development, unrelated to the old format. + // new development, unrelated to the old format. stataReleaseNumber.put((byte) 117, "rel_13"); readableFileTypes.add("application/x-stata"); @@ -153,14 +153,14 @@ public String testSAVformat(MappedByteBuffer buff) { buff.rewind(); boolean DEBUG = false; - + // ----------------------------------------- // Avoid java.nio.BufferUnderflowException // ----------------------------------------- if (buff.capacity() < 4){ return null; } - + if (DEBUG) { out.println("applying the sav test\n"); } @@ -182,11 +182,11 @@ public String testSAVformat(MappedByteBuffer buff) { out.println("this file is NOT spss-sav type"); } } - + return result; } - + /** * test this byte buffer against STATA DTA spec * @@ -203,16 +203,16 @@ public String testDTAformat(MappedByteBuffer buff) { // ----------------------------------------- // Avoid java.nio.BufferUnderflowException // ----------------------------------------- - if (buff.capacity() < 4) { + if (buff.capacity() < 4) { return result; } - - // We first check if it's a "classic", old DTA format - // (up to version 115): - + + // We first check if it's a "classic", old DTA format + // (up to version 115): + byte[] hdr4 = new byte[4]; buff.get(hdr4, 0, 4); - + if (DEBUG) { for (int i = 0; i < hdr4.length; ++i) { dbgLog.info(String.format("%d\t%02X\n", i, hdr4[i])); @@ -243,26 +243,26 @@ public String testDTAformat(MappedByteBuffer buff) { } result = "application/x-stata"; } - + if ((result == null)&&(buff.capacity() >= STATA_13_HEADER.length())) { - // Let's see if it's a "new" STATA (v.13+) format: + // Let's see if it's a "new" STATA (v.13+) format: buff.rewind(); - byte[] headerBuffer = null; - String headerString = null; + byte[] headerBuffer = null; + String headerString = null; try { headerBuffer = new byte[STATA_13_HEADER.length()]; buff.get(headerBuffer, 0, STATA_13_HEADER.length()); headerString = new String(headerBuffer, "US-ASCII"); } catch (Exception ex) { - // probably a buffer underflow exception; - // we don't have to do anything... null will - // be returned, below. + // probably a buffer underflow exception; + // we don't have to do anything... null will + // be returned, below. } - + if (STATA_13_HEADER.equals(headerString)) { result = "application/x-stata-13"; } - + } if ((result == null) && (buff.capacity() >= STATA_14_HEADER.length())) { @@ -368,8 +368,8 @@ public String testPORformat(MappedByteBuffer buff) { } // size test - int bufferCapacity = buff.capacity(); - dbgLog.fine("Subsettable Checker: buffer capacity: "+bufferCapacity); + int bufferCapacity = buff.capacity(); + dbgLog.fine("Subsettable Checker: buffer capacity: "+bufferCapacity); if (bufferCapacity < 491) { if (DEBUG) { @@ -404,10 +404,10 @@ public String testPORformat(MappedByteBuffer buff) { // 1-char case pos1 = baseBias + i; - if ( pos1 > bufferCapacity - 1 ) { - dbgLog.fine("Subsettable Checker: request to go beyond buffer capacity ("+pos1+")"); - return result; - } + if ( pos1 > bufferCapacity - 1 ) { + dbgLog.fine("Subsettable Checker: request to go beyond buffer capacity ("+pos1+")"); + return result; + } buff.position(pos1); if (DEBUG) { @@ -425,10 +425,10 @@ public String testPORformat(MappedByteBuffer buff) { // 2-char case pos2 = baseBias + 2 * i; - if ( pos2 > bufferCapacity - 2 ) { - dbgLog.fine("Subsettable Checker: request to read 2 bytes beyond buffer capacity ("+pos2+")"); - return result; - } + if ( pos2 > bufferCapacity - 2 ) { + dbgLog.fine("Subsettable Checker: request to read 2 bytes beyond buffer capacity ("+pos2+")"); + return result; + } buff.position(pos2); @@ -441,10 +441,10 @@ public String testPORformat(MappedByteBuffer buff) { // 3-char case pos3 = baseBias + 3 * i; - if ( pos3 > bufferCapacity - 3 ) { - dbgLog.fine("Subsettable Checker: request to read 3 bytes beyond buffer capacity ("+pos3+")"); - return result; - } + if ( pos3 > bufferCapacity - 3 ) { + dbgLog.fine("Subsettable Checker: request to read 3 bytes beyond buffer capacity ("+pos3+")"); + return result; + } buff.position(pos3); @@ -527,11 +527,11 @@ public String testPORformat(MappedByteBuffer buff) { public String testRDAformat(MappedByteBuffer buff) { String result = null; buff.rewind(); - + if (buff.capacity() < 4){ return null; } - + boolean DEBUG = false; if (DEBUG) { out.println("applying the RData test\n"); @@ -546,7 +546,7 @@ public String testRDAformat(MappedByteBuffer buff) { out.println(); buff.rewind(); } - // get the first 4 bytes as an int and check its value; + // get the first 4 bytes as an int and check its value; // if it is 0x1F8B0800, then gunzip and its first 4 bytes int magicNumber = buff.getInt(); @@ -596,7 +596,7 @@ public String testRDAformat(MappedByteBuffer buff) { String fisrt5bytes = sb.toString(); result = this.checkUncompressedFirst5bytes(fisrt5bytes); - // end of uncompressed case + // end of uncompressed case } } catch (IOException ex) { ex.printStackTrace(); @@ -613,7 +613,7 @@ public String detectTabularDataFormat(File fh) { try { int buffer_size = this.getBufferSize(fh); dbgLog.fine("buffer_size: " + buffer_size); - + // set-up a FileChannel instance for a given file object inp = new FileInputStream(fh); srcChannel = inp.getChannel(); @@ -627,7 +627,7 @@ public String detectTabularDataFormat(File fh) { buff.rewind(); dbgLog.fine("before the for loop"); for (String fmt : this.getTestFormatSet()) { - + // get a test method Method mthd = testMethods.get(fmt); //dbgLog.info("mthd: " + mthd.getName()); @@ -713,7 +713,7 @@ private String checkUncompressedFirst5bytes(String fisrt5bytes) { } /** - * adjust the size of the buffer according to the size of + * adjust the size of the buffer according to the size of * the file if necessary; otherwise, use the default size */ private int getBufferSize(File fh) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index ee9ad91bdf9..3c890e53db1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -83,27 +83,26 @@ * a 4.0 implementation of the DVN FileUtil; * it provides some of the functionality from the 3.6 implementation, * but the old code is ported creatively on the method-by-method basis. - * + * * @author Leonid Andreev */ public class FileUtil implements java.io.Serializable { private static final Logger logger = Logger.getLogger(FileUtil.class.getCanonicalName()); - + private static final String[] TABULAR_DATA_FORMAT_SET = {"POR", "SAV", "DTA", "RDA"}; - + private static Map STATISTICAL_FILE_EXTENSION = new HashMap(); - /* - * The following are Stata, SAS and SPSS syntax/control cards: - * These are recognized as text files (because they are!) so - * we check all the uploaded "text/plain" files for these extensions, and + * The following are Stata, SAS and SPSS syntax/control cards: + * These are recognized as text files (because they are!) so + * we check all the uploaded "text/plain" files for these extensions, and * assign the following types when they are matched; - * Note that these types are only used in the metadata displayed on the - * dataset page. We don't support ingest on control cards. + * Note that these types are only used in the metadata displayed on the + * dataset page. We don't support ingest on control cards. * -- L.A. 4.0 Oct. 2014 - */ - + */ + static { STATISTICAL_FILE_EXTENSION.put("do", "application/x-stata-syntax"); STATISTICAL_FILE_EXTENSION.put("sas", "application/x-sas-syntax"); @@ -111,9 +110,9 @@ public class FileUtil implements java.io.Serializable { STATISTICAL_FILE_EXTENSION.put("csv", "text/csv"); STATISTICAL_FILE_EXTENSION.put("tsv", "text/tsv"); } - + private static MimetypesFileTypeMap MIME_TYPE_MAP = new MimetypesFileTypeMap(); - + public static final String MIME_TYPE_STATA = "application/x-stata"; public static final String MIME_TYPE_STATA13 = "application/x-stata-13"; public static final String MIME_TYPE_STATA14 = "application/x-stata-14"; @@ -129,20 +128,20 @@ public class FileUtil implements java.io.Serializable { public static final String MIME_TYPE_SPSS_SAV = "application/x-spss-sav"; public static final String MIME_TYPE_SPSS_POR = "application/x-spss-por"; - + public static final String MIME_TYPE_FITS = "application/fits"; - + public static final String MIME_TYPE_ZIP = "application/zip"; - + public static final String MIME_TYPE_FITSIMAGE = "image/fits"; - // SHAPE file type: + // SHAPE file type: // this is the only supported file type in the GEO DATA class: - + public static final String MIME_TYPE_GEO_SHAPE = "application/zipped-shapefile"; - + public static final String MIME_TYPE_UNDETERMINED_DEFAULT = "application/octet-stream"; public static final String MIME_TYPE_UNDETERMINED_BINARY = "application/binary"; - + public static final String SAVED_ORIGINAL_FILENAME_EXTENSION = "orig"; public static final String MIME_TYPE_INGESTED_FILE = "text/tab-separated-values"; @@ -157,11 +156,11 @@ public class FileUtil implements java.io.Serializable { public FileUtil() { } - + public static void copyFile(File inputFile, File outputFile) throws IOException { FileChannel in = null; WritableByteChannel out = null; - + try { in = new FileInputStream(inputFile).getChannel(); out = new FileOutputStream(outputFile).getChannel(); @@ -171,26 +170,26 @@ public static void copyFile(File inputFile, File outputFile) throws IOException in.transferTo(start, bytesPerIteration, out); start += bytesPerIteration; } - + } finally { if (in != null) { in.close(); } if (out != null) { out.close(); } } } - + public static String getFileExtension(String fileName){ String ext = null; if ( fileName.lastIndexOf(".") != -1){ ext = (fileName.substring( fileName.lastIndexOf(".") + 1 )).toLowerCase(); } return ext; - } + } public static String replaceExtension(String originalName) { - return replaceExtension(originalName, "tab"); - } - + return replaceExtension(originalName, "tab"); + } + public static String replaceExtension(String originalName, String newExtension) { int extensionIndex = originalName.lastIndexOf("."); if (extensionIndex != -1 ) { @@ -199,10 +198,10 @@ public static String replaceExtension(String originalName, String newExtension) return originalName +"."+newExtension ; } } - + public static String getUserFriendlyFileType(DataFile dataFile) { String fileType = dataFile.getContentType(); - + if (fileType != null) { if (fileType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)){ return ShapefileHandler.SHAPEFILE_FILE_TYPE_FRIENDLY_NAME; @@ -219,10 +218,9 @@ public static String getUserFriendlyFileType(DataFile dataFile) { return fileType; } - + public static String getFacetFileType(DataFile dataFile) { String fileType = dataFile.getContentType(); - if (!StringUtil.isEmpty(fileType)) { if (fileType.contains(";")) { @@ -233,11 +231,11 @@ public static String getFacetFileType(DataFile dataFile) { return BundleUtil.getStringFromPropertyFile(fileType,"MimeTypeFacets" ); } catch (MissingResourceException e) { // if there's no defined "facet-friendly" form of this mime type - // we'll truncate the available type by "/", e.g., all the + // we'll truncate the available type by "/", e.g., all the // unknown image/* types will become "image"; many other, quite - // different types will all become "application" this way - - // but it is probably still better than to tag them all as - // "uknown". + // different types will all become "application" this way - + // but it is probably still better than to tag them all as + // "uknown". // -- L.A. 4.0 alpha 1 // // UPDATE, MH 4.9.2 @@ -258,14 +256,14 @@ public static String getFacetFileType(DataFile dataFile) { } } } - + public static String getUserFriendlyOriginalType(DataFile dataFile) { if (!dataFile.isTabularData()) { return null; } String fileType = dataFile.getOriginalFileFormat(); - + if (fileType != null && !fileType.equals("")) { if (fileType.contains(";")) { fileType = fileType.substring(0, fileType.indexOf(";")); @@ -275,14 +273,14 @@ public static String getUserFriendlyOriginalType(DataFile dataFile) { } catch (MissingResourceException e) { return fileType; } - } - + } + return "UNKNOWN"; } - + /** * Returns a content type string for a FileObject - * + * */ private static String determineContentType(File fileObject) { if (fileObject==null){ @@ -296,13 +294,13 @@ private static String determineContentType(File fileObject) { contentType = null; } - if ((contentType==null)||(contentType.equals(""))){ + if ((contentType==null)||(contentType.equals(""))){ contentType = MIME_TYPE_UNDETERMINED_DEFAULT; - } - return contentType; - + } + return contentType; + } - + public static String retestIngestableFileType(File file, String fileType) { IngestableDataChecker tabChecker = new IngestableDataChecker(TABULAR_DATA_FORMAT_SET); String newType = tabChecker.detectTabularDataFormat(file); @@ -322,28 +320,28 @@ public static String determineFileType(File f, String fileName) throws IOExcepti // (such as FITS). logger.fine("Attempting to identify potential tabular data files;"); IngestableDataChecker tabChk = new IngestableDataChecker(TABULAR_DATA_FORMAT_SET); - + fileType = tabChk.detectTabularDataFormat(f); - + logger.fine("determineFileType: tabular data checker found "+fileType); - + // step 2: If not found, check if graphml or FITS if (fileType==null) { if (isGraphMLFile(f)) { fileType = "text/xml-graphml"; } else // Check for FITS: - // our check is fairly weak (it appears to be hard to really - // really recognize a FITS file without reading the entire - // stream...), so in version 3.* we used to nsist on *both* - // the ".fits" extension and the header check; - // in 4.0, we'll accept either the extension, or the valid - // magic header: - if (isFITSFile(f) || (fileExtension != null + // our check is fairly weak (it appears to be hard to really + // really recognize a FITS file without reading the entire + // stream...), so in version 3.* we used to nsist on *both* + // the ".fits" extension and the header check; + // in 4.0, we'll accept either the extension, or the valid + // magic header: + if (isFITSFile(f) || (fileExtension != null && fileExtension.equalsIgnoreCase("fits"))) { - fileType = "application/fits"; - } + fileType = "application/fits"; + } } - + // step 3: check the mime type of this file with Jhove if (fileType == null){ JhoveFileType jw = new JhoveFileType(); @@ -352,12 +350,12 @@ public static String determineFileType(File f, String fileName) throws IOExcepti fileType = mimeType; } } - - // step 4: - // Additional processing; if we haven't gotten much useful information - // back from Jhove, we'll try and make an educated guess based on + + // step 4: + // Additional processing; if we haven't gotten much useful information + // back from Jhove, we'll try and make an educated guess based on // the file extension: - + if ( fileExtension != null) { logger.fine("fileExtension="+fileExtension); @@ -367,15 +365,15 @@ public static String determineFileType(File f, String fileName) throws IOExcepti } else { fileType = determineFileTypeByExtension(fileName); } - + logger.fine("mime type recognized by extension: "+fileType); } } else { logger.fine("fileExtension is null"); } - - // step 5: - // if this is a compressed file - zip or gzip - we'll check the + + // step 5: + // if this is a compressed file - zip or gzip - we'll check the // file(s) inside the compressed stream and see if it's one of our // recognized formats that we want to support compressed: @@ -385,9 +383,9 @@ public static String determineFileType(File f, String fileName) throws IOExcepti // if they were just regular FITS files: FileInputStream gzippedIn = new FileInputStream(f); // (new FileInputStream() can throw a "filen not found" exception; - // however, if we've made it this far, it really means that the + // however, if we've made it this far, it really means that the // file does exist and can be opened) - InputStream uncompressedIn = null; + InputStream uncompressedIn = null; try { uncompressedIn = new GZIPInputStream(gzippedIn); if (isFITSFile(uncompressedIn)) { @@ -398,38 +396,38 @@ public static String determineFileType(File f, String fileName) throws IOExcepti try {uncompressedIn.close();} catch (IOException e) {} } } - } + } if ("application/zip".equals(fileType)) { - + // Is this a zipped Shapefile? // Check for shapefile extensions as described here: http://en.wikipedia.org/wiki/Shapefile //logger.info("Checking for shapefile"); ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(f)); - if (shp_handler.containsShapefile()){ - // logger.info("------- shapefile FOUND ----------"); - fileType = ShapefileHandler.SHAPEFILE_FILE_TYPE; //"application/zipped-shapefile"; - } - } - + if (shp_handler.containsShapefile()){ + // logger.info("------- shapefile FOUND ----------"); + fileType = ShapefileHandler.SHAPEFILE_FILE_TYPE; //"application/zipped-shapefile"; + } + } + logger.fine("returning fileType "+fileType); return fileType; } - + public static String determineFileTypeByExtension(String fileName) { logger.fine("Type by extension, for "+fileName+": "+MIME_TYPE_MAP.getContentType(fileName)); return MIME_TYPE_MAP.getContentType(fileName); } - - - /* - * Custom method for identifying FITS files: - * TODO: - * the existing check for the "magic header" is very weak (see below); - * it should probably be replaced by attempting to parse and read at - * least the primary HDU, using the NOM fits parser. + + + /* + * Custom method for identifying FITS files: + * TODO: + * the existing check for the "magic header" is very weak (see below); + * it should probably be replaced by attempting to parse and read at + * least the primary HDU, using the NOM fits parser. * -- L.A. 4.0 alpha - */ + */ private static boolean isFITSFile(File file) { BufferedInputStream ins = null; @@ -437,15 +435,15 @@ private static boolean isFITSFile(File file) { ins = new BufferedInputStream(new FileInputStream(file)); return isFITSFile(ins); } catch (IOException ex) { - } - + } + return false; } - + private static boolean isFITSFile(InputStream ins) { boolean isFITS = false; - // number of header bytes read for identification: + // number of header bytes read for identification: int magicWordLength = 6; String magicWord = "SIMPLE"; @@ -461,7 +459,7 @@ private static boolean isFITSFile(InputStream ins) { isFITS = true; } } catch (IOException ex) { - isFITS = false; + isFITS = false; } finally { if (ins != null) { try { @@ -470,10 +468,10 @@ private static boolean isFITSFile(InputStream ins) { } } } - + return isFITS; } - + private static boolean isGraphMLFile(File file) { boolean isGraphML = false; logger.fine("begin isGraphMLFile()"); @@ -571,73 +569,73 @@ public static String generateOriginalExtension(String fileType) { return ""; } - + public static List createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, SystemConfig systemConfig) throws IOException { - List datafiles = new ArrayList<>(); - - String warningMessage = null; - - // save the file, in the temporary location for now: - Path tempFile = null; - + List datafiles = new ArrayList<>(); + + String warningMessage = null; + + // save the file, in the temporary location for now: + Path tempFile = null; + Long fileSizeLimit = systemConfig.getMaxFileUploadSize(); - + if (getFilesTempDirectory() != null) { tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); - // "temporary" location is the key here; this is why we are not using - // the DataStore framework for this - the assumption is that - // temp files will always be stored on the local filesystem. + // "temporary" location is the key here; this is why we are not using + // the DataStore framework for this - the assumption is that + // temp files will always be stored on the local filesystem. // -- L.A. Jul. 2014 logger.fine("Will attempt to save the file as: " + tempFile.toString()); Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); - + // A file size check, before we do anything else: // (note that "no size limit set" = "unlimited") - // (also note, that if this is a zip file, we'll be checking + // (also note, that if this is a zip file, we'll be checking // the size limit for each of the individual unpacked files) Long fileSize = tempFile.toFile().length(); if (fileSizeLimit != null && fileSize > fileSizeLimit) { try {tempFile.toFile().delete();} catch (Exception ex) {} - throw new IOException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); + throw new IOException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); } - + } else { throw new IOException ("Temp directory is not configured."); } logger.fine("mime type supplied: "+suppliedContentType); - // Let's try our own utilities (Jhove, etc.) to determine the file type + // Let's try our own utilities (Jhove, etc.) to determine the file type // of the uploaded file. (We may already have a mime type supplied for this - // file - maybe the type that the browser recognized on upload; or, if + // file - maybe the type that the browser recognized on upload; or, if // it's a harvest, maybe the remote server has already given us the type - // for this file... with our own type utility we may or may not do better + // for this file... with our own type utility we may or may not do better // than the type supplied: - // -- L.A. + // -- L.A. String recognizedType = null; - String finalType = null; + String finalType = null; try { recognizedType = determineFileType(tempFile.toFile(), fileName); logger.fine("File utility recognized the file as " + recognizedType); if (recognizedType != null && !recognizedType.equals("")) { // is it any better than the type that was supplied to us, // if any? - // This is not as trivial a task as one might expect... + // This is not as trivial a task as one might expect... // We may need a list of "good" mime types, that should always - // be chosen over other choices available. Maybe it should - // even be a weighed list... as in, "application/foo" should + // be chosen over other choices available. Maybe it should + // even be a weighed list... as in, "application/foo" should // be chosen over "application/foo-with-bells-and-whistles". - - // For now the logic will be as follows: + + // For now the logic will be as follows: // - // 1. If the contentType supplied (by the browser, most likely) - // is some form of "unknown", we always discard it in favor of - // whatever our own utilities have determined; - // 2. We should NEVER trust the browser when it comes to the + // 1. If the contentType supplied (by the browser, most likely) + // is some form of "unknown", we always discard it in favor of + // whatever our own utilities have determined; + // 2. We should NEVER trust the browser when it comes to the // following "ingestable" types: Stata, SPSS, R; // 2a. We are willing to TRUST the browser when it comes to // the CSV and XSLX ingestable types. - // 3. We should ALWAYS trust our utilities when it comes to - // ingestable types. - + // 3. We should ALWAYS trust our utilities when it comes to + // ingestable types. + if (suppliedContentType == null || suppliedContentType.equals("") || suppliedContentType.equalsIgnoreCase(MIME_TYPE_UNDETERMINED_DEFAULT) @@ -653,34 +651,34 @@ public static List createDataFiles(DatasetVersion version, InputStream finalType = recognizedType; } } - + } catch (Exception ex) { logger.warning("Failed to run the file utility mime type check on file " + fileName); } - + if (finalType == null) { - finalType = (suppliedContentType == null || suppliedContentType.equals("")) + finalType = (suppliedContentType == null || suppliedContentType.equals("")) ? MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; } - - // A few special cases: - + + // A few special cases: + // if this is a gzipped FITS file, we'll uncompress it, and ingest it as // a regular FITS file: - + if (finalType.equals("application/fits-gzipped")) { InputStream uncompressedIn = null; String finalFileName = fileName; - // if the file name had the ".gz" extension, remove it, + // if the file name had the ".gz" extension, remove it, // since we are going to uncompress it: if (fileName != null && fileName.matches(".*\\.gz$")) { finalFileName = fileName.replaceAll("\\.gz$", ""); } - - DataFile datafile = null; - try { + + DataFile datafile = null; + try { uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile())); File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit); datafile = createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, systemConfig.getFileFixityChecksumAlgorithm()); @@ -691,45 +689,45 @@ public static List createDataFiles(DatasetVersion version, InputStream try {uncompressedIn.close();} catch (IOException e) {} } } - - // If we were able to produce an uncompressed file, we'll use it + + // If we were able to produce an uncompressed file, we'll use it // to create and return a final DataFile; if not, we're not going - // to do anything - and then a new DataFile will be created further + // to do anything - and then a new DataFile will be created further // down, from the original, uncompressed file. if (datafile != null) { - // remove the compressed temp file: + // remove the compressed temp file: try { tempFile.toFile().delete(); } catch (SecurityException ex) { // (this is very non-fatal) logger.warning("Failed to delete temporary file "+tempFile.toString()); } - + datafiles.add(datafile); return datafiles; } - - // If it's a ZIP file, we are going to unpack it and create multiple - // DataFile objects from its contents: - } else if (finalType.equals("application/zip")) { - - ZipInputStream unZippedIn = null; - ZipEntry zipEntry = null; - + + // If it's a ZIP file, we are going to unpack it and create multiple + // DataFile objects from its contents: + } else if (finalType.equals("application/zip")) { + + ZipInputStream unZippedIn = null; + ZipEntry zipEntry = null; + int fileNumberLimit = systemConfig.getZipUploadFilesLimit(); - + try { Charset charset = null; /* TODO: (?) - We may want to investigate somehow letting the user specify + We may want to investigate somehow letting the user specify the charset for the filenames in the zip file... - - otherwise, ZipInputStream bails out if it encounteres a file - name that's not valid in the current charest (i.e., UTF-8, in - our case). It would be a bit trickier than what we're doing for - SPSS tabular ingests - with the lang. encoding pulldown menu - + - otherwise, ZipInputStream bails out if it encounteres a file + name that's not valid in the current charest (i.e., UTF-8, in + our case). It would be a bit trickier than what we're doing for + SPSS tabular ingests - with the lang. encoding pulldown menu - because this encoding needs to be specified *before* we upload and - attempt to unzip the file. + attempt to unzip the file. -- L.A. 4.0 beta12 logger.info("default charset is "+Charset.defaultCharset().name()); if (Charset.isSupported("US-ASCII")) { @@ -738,41 +736,41 @@ public static List createDataFiles(DatasetVersion version, InputStream if (charset != null) { logger.info("was able to obtain charset for US-ASCII"); } - + } */ - + if (charset != null) { unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset); } else { unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile())); - } + } - while (true) { + while (true) { try { zipEntry = unZippedIn.getNextEntry(); } catch (IllegalArgumentException iaex) { - // Note: - // ZipInputStream documentation doesn't even mention that + // Note: + // ZipInputStream documentation doesn't even mention that // getNextEntry() throws an IllegalArgumentException! // but that's what happens if the file name of the next - // entry is not valid in the current CharSet. + // entry is not valid in the current CharSet. // -- L.A. warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is."; logger.warning(warningMessage); throw new IOException(); - } - + } + if (zipEntry == null) { break; } - // Note that some zip entries may be directories - we + // Note that some zip entries may be directories - we // simply skip them: - + if (!zipEntry.isDirectory()) { if (datafiles.size() > fileNumberLimit) { logger.warning("Zip upload - too many files."); - warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + + warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit + "); please upload a zip archive with fewer files, if you want them to be ingested " + "as individual DataFiles."; throw new IOException(); @@ -785,11 +783,11 @@ public static List createDataFiles(DatasetVersion version, InputStream String shortName = fileEntryName.replaceFirst("^.*[\\/]", ""); - // Check if it's a "fake" file - a zip archive entry - // created for a MacOS X filesystem element: (these + // Check if it's a "fake" file - a zip archive entry + // created for a MacOS X filesystem element: (these // start with "._") if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) { - // OK, this seems like an OK file entry - we'll try + // OK, this seems like an OK file entry - we'll try // to read it and create a DataFile with it: File unZippedTempFile = saveInputStreamInTempFile(unZippedIn, fileSizeLimit); @@ -797,8 +795,8 @@ public static List createDataFiles(DatasetVersion version, InputStream if (!fileEntryName.equals(shortName)) { // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes), - // we'll extract the directory name, then a) strip the leading and trailing slashes; - // and b) replace all the back slashes with regular ones and b) replace any multiple + // we'll extract the directory name, then a) strip the leading and trailing slashes; + // and b) replace all the back slashes with regular ones and b) replace any multiple // slashes with a single slash: String directoryName = fileEntryName.replaceFirst("[\\/][\\/]*[^\\/]*$", "").replaceFirst("^[\\/]*", "").replaceAll("[\\/][\\/]*", "/"); if (!"".equals(directoryName)) { @@ -806,42 +804,42 @@ public static List createDataFiles(DatasetVersion version, InputStream datafile.getFileMetadata().setDirectoryLabel(directoryName); } } - + if (datafile != null) { // We have created this datafile with the mime type "unknown"; - // Now that we have it saved in a temporary location, + // Now that we have it saved in a temporary location, // let's try and determine its real type: - + String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(); - + try { recognizedType = determineFileType(new File(tempFileName), shortName); logger.fine("File utility recognized unzipped file as " + recognizedType); if (recognizedType != null && !recognizedType.equals("")) { - datafile.setContentType(recognizedType); + datafile.setContentType(recognizedType); } } catch (Exception ex) { logger.warning("Failed to run the file utility mime type check on file " + fileName); } - + datafiles.add(datafile); } } } - } - unZippedIn.closeEntry(); - + } + unZippedIn.closeEntry(); + } - + } catch (IOException ioex) { - // just clear the datafiles list and let + // just clear the datafiles list and let // ingest default to creating a single DataFile out - // of the unzipped file. + // of the unzipped file. logger.warning("Unzipping failed; rolling back to saving the file as is."); if (warningMessage == null) { warningMessage = "Failed to unzip the file. Saving the file as is."; } - + datafiles.clear(); } catch (FileExceedsMaxSizeException femsx) { logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage()); @@ -853,7 +851,7 @@ public static List createDataFiles(DatasetVersion version, InputStream } } if (datafiles.size() > 0) { - // link the data files to the dataset/version: + // link the data files to the dataset/version: // (except we no longer want to do this! -- 4.6) /*Iterator itf = datafiles.iterator(); while (itf.hasNext()) { @@ -864,10 +862,10 @@ public static List createDataFiles(DatasetVersion version, InputStream } version.getFileMetadatas().add(datafile.getFileMetadata()); datafile.getFileMetadata().setDatasetVersion(version); - + version.getDataset().getFiles().add(datafile); } */ - // remove the uploaded zip file: + // remove the uploaded zip file: try { Files.delete(tempFile); } catch (IOException ioex) { @@ -877,23 +875,23 @@ public static List createDataFiles(DatasetVersion version, InputStream // and return: return datafiles; } - + } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) { - // Shape files may have to be split into multiple files, + // Shape files may have to be split into multiple files, // one zip archive per each complete set of shape files: - + //File rezipFolder = new File(this.getFilesTempDirectory()); File rezipFolder = getShapefileUnzipTempDirectory(); - + IngestServiceShapefileHelper shpIngestHelper; shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder); boolean didProcessWork = shpIngestHelper.processFile(); - if (!(didProcessWork)){ + if (!(didProcessWork)){ logger.severe("Processing of zipped shapefile failed."); return null; } - + try { for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) { FileInputStream finalFileInputStream = new FileInputStream(finalFile); @@ -917,7 +915,7 @@ public static List createDataFiles(DatasetVersion version, InputStream logger.severe("One of the unzipped shape files exceeded the size limit; giving up. " + femsx.getMessage()); datafiles.clear(); } - + // Delete the temp directory used for unzipping FileUtils.deleteDirectory(rezipFolder); @@ -937,63 +935,63 @@ public static List createDataFiles(DatasetVersion version, InputStream logger.severe("No files added from directory of rezipped shapefiles"); } return null; - - } - // Finally, if none of the special cases above were applicable (or - // if we were unable to unpack an uploaded file, etc.), we'll just + + } + // Finally, if none of the special cases above were applicable (or + // if we were unable to unpack an uploaded file, etc.), we'll just // create and return a single DataFile: - + DataFile datafile = createSingleDataFile(version, tempFile.toFile(), fileName, finalType, systemConfig.getFileFixityChecksumAlgorithm()); - + if (datafile != null && tempFile.toFile() != null) { - + if (warningMessage != null) { createIngestFailureReport(datafile, warningMessage); datafile.SetIngestProblem(); } datafiles.add(datafile); - + return datafiles; } - + return null; } // end createDataFiles - + private static File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit) throws IOException, FileExceedsMaxSizeException { Path tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload"); - + if (inputStream != null && tempFile != null) { Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); - - // size check: + + // size check: // (note that "no size limit set" = "unlimited") Long fileSize = tempFile.toFile().length(); if (fileSizeLimit != null && fileSize > fileSizeLimit) { try {tempFile.toFile().delete();} catch (Exception ex) {} - throw new FileExceedsMaxSizeException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); + throw new FileExceedsMaxSizeException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit))); } - + return tempFile.toFile(); } throw new IOException("Failed to save uploaded file."); } - - /* - * This method creates a DataFile; - * The bytes from the suppplied InputStream have already been saved in the temporary location. - * This method should only be called by the upper-level methods that handle - * file upload and creation for individual use cases - a single file upload, - * an upload of a zip archive that needs to be unpacked and turned into - * individual files, etc., and once the file name and mime type have already - * been figured out. - */ - + + /* + * This method creates a DataFile; + * The bytes from the suppplied InputStream have already been saved in the temporary location. + * This method should only be called by the upper-level methods that handle + * file upload and creation for individual use cases - a single file upload, + * an upload of a zip archive that needs to be unpacked and turned into + * individual files, etc., and once the file name and mime type have already + * been figured out. + */ + private static DataFile createSingleDataFile(DatasetVersion version, File tempFile, String fileName, String contentType, DataFile.ChecksumType checksumType) { return createSingleDataFile(version, tempFile, fileName, contentType, checksumType, false); } - + private static DataFile createSingleDataFile(DatasetVersion version, File tempFile, String fileName, String contentType, DataFile.ChecksumType checksumType, boolean addToDataset) { if (tempFile == null) { @@ -1043,16 +1041,16 @@ private static DataFile createSingleDataFile(DatasetVersion version, File tempFi return datafile; } - - + + /** - For the restructuring of zipped shapefiles, create a timestamped directory. - This directory is deleted after successful restructuring. - - Naming convention: getFilesTempDirectory() + "shp_" + "yyyy-MM-dd-hh-mm-ss-SSS" - */ + For the restructuring of zipped shapefiles, create a timestamped directory. + This directory is deleted after successful restructuring. + + Naming convention: getFilesTempDirectory() + "shp_" + "yyyy-MM-dd-hh-mm-ss-SSS" + */ private static File getShapefileUnzipTempDirectory(){ - + String tempDirectory = getFilesTempDirectory(); if (tempDirectory == null){ logger.severe("Failed to retrieve tempDirectory, null was returned" ); @@ -1060,12 +1058,12 @@ private static File getShapefileUnzipTempDirectory(){ } String datestampedFileName = "shp_" + new SimpleDateFormat("yyyy-MM-dd-hh-mm-ss-SSS").format(new Date()); String datestampedFolderName = tempDirectory + "/" + datestampedFileName; - + File datestampedFolder = new File(datestampedFolderName); if (!datestampedFolder.isDirectory()) { - /* Note that "createDirectories()" must be used - not - * "createDirectory()", to make sure all the parent - * directories that may not yet exist are created as well. + /* Note that "createDirectories()" must be used - not + * "createDirectory()", to make sure all the parent + * directories that may not yet exist are created as well. */ try { Files.createDirectories(Paths.get(datestampedFolderName)); @@ -1074,9 +1072,9 @@ private static File getShapefileUnzipTempDirectory(){ return null; } } - return datestampedFolder; + return datestampedFolder; } - + public static boolean canIngestAsTabular(DataFile dataFile) { String mimeType = dataFile.getContentType(); @@ -1091,11 +1089,11 @@ public static boolean canIngestAsTabular(String mimeType) { * main code base, so we can just go through a hard-coded list of mime * types. -- L.A. */ - + if (mimeType == null) { return false; } - + switch (mimeType) { case MIME_TYPE_STATA: case MIME_TYPE_STATA13: @@ -1114,7 +1112,7 @@ public static boolean canIngestAsTabular(String mimeType) { return false; } } - + public static String getFilesTempDirectory() { String filesRootDirectory = System.getProperty("dataverse.files.directory"); if (filesRootDirectory == null || filesRootDirectory.equals("")) { @@ -1124,9 +1122,9 @@ public static String getFilesTempDirectory() { String filesTempDirectory = filesRootDirectory + "/temp"; if (!Files.exists(Paths.get(filesTempDirectory))) { - /* Note that "createDirectories()" must be used - not - * "createDirectory()", to make sure all the parent - * directories that may not yet exist are created as well. + /* Note that "createDirectories()" must be used - not + * "createDirectory()", to make sure all the parent + * directories that may not yet exist are created as well. */ try { Files.createDirectories(Paths.get(filesTempDirectory)); @@ -1138,7 +1136,7 @@ public static String getFilesTempDirectory() { return filesTempDirectory; } - + public static void generateS3PackageStorageIdentifier(DataFile dataFile) { String bucketName = System.getProperty("dataverse.files.s3-bucket-name"); String storageId = S3_IDENTIFIER_PREFIX + "://" + bucketName + ":" + dataFile.getFileMetadata().getLabel(); @@ -1148,40 +1146,40 @@ public static void generateS3PackageStorageIdentifier(DataFile dataFile) { public static void generateStorageIdentifier(DataFile dataFile) { dataFile.setStorageIdentifier(generateStorageIdentifier()); } - + public static String generateStorageIdentifier() { - + UUID uid = UUID.randomUUID(); - + logger.log(Level.FINE, "UUID value: {0}", uid.toString()); - - // last 6 bytes, of the random UUID, in hex: - + + // last 6 bytes, of the random UUID, in hex: + String hexRandom = uid.toString().substring(24); - + logger.log(Level.FINE, "UUID (last 6 bytes, 12 hex digits): {0}", hexRandom); - + String hexTimestamp = Long.toHexString(new Date().getTime()); - + logger.log(Level.FINE, "(not UUID) timestamp in hex: {0}", hexTimestamp); - + String storageIdentifier = hexTimestamp + "-" + hexRandom; - + logger.log(Level.FINE, "timestamp/UUID hybrid: {0}", storageIdentifier); - return storageIdentifier; + return storageIdentifier; } - + public static void createIngestFailureReport(DataFile dataFile, String message) { createIngestReport(dataFile, IngestReport.INGEST_STATUS_FAILURE, message); } - + private static void createIngestReport (DataFile dataFile, int status, String message) { IngestReport errorReport = new IngestReport(); if (status == IngestReport.INGEST_STATUS_FAILURE) { - errorReport.setFailure(); - errorReport.setReport(message); - errorReport.setDataFile(dataFile); - dataFile.setIngestReport(errorReport); + errorReport.setFailure(); + errorReport.setReport(message); + errorReport.setDataFile(dataFile); + dataFile.setIngestReport(errorReport); } } @@ -1215,8 +1213,8 @@ public static String getCiteDataFileFilename(String fileTitle, FileCitationExten * elaborate on the text "This file cannot be downloaded publicly." */ public static boolean isDownloadPopupRequired(DatasetVersion datasetVersion) { - // Each of these conditions is sufficient reason to have to - // present the user with the popup: + // Each of these conditions is sufficient reason to have to + // present the user with the popup: if (datasetVersion == null) { logger.fine("Download popup required because datasetVersion is null."); return false; @@ -1229,8 +1227,8 @@ public static boolean isDownloadPopupRequired(DatasetVersion datasetVersion) { // 1. License and Terms of Use: if (datasetVersion.getTermsOfUseAndAccess() != null) { if (!TermsOfUseAndAccess.License.CC0.equals(datasetVersion.getTermsOfUseAndAccess().getLicense()) - && !(datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null - || datasetVersion.getTermsOfUseAndAccess().getTermsOfUse().equals(""))) { + && !(datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null + || datasetVersion.getTermsOfUseAndAccess().getTermsOfUse().equals(""))) { logger.fine("Download popup required because of license or terms of use."); return true; } @@ -1251,10 +1249,10 @@ public static boolean isDownloadPopupRequired(DatasetVersion datasetVersion) { logger.fine("Download popup is not required."); return false; } - + public static boolean isRequestAccessPopupRequired(DatasetVersion datasetVersion){ - // Each of these conditions is sufficient reason to have to - // present the user with the popup: + // Each of these conditions is sufficient reason to have to + // present the user with the popup: if (datasetVersion == null) { logger.fine("Download popup required because datasetVersion is null."); return false; @@ -1267,8 +1265,8 @@ public static boolean isRequestAccessPopupRequired(DatasetVersion datasetVersion // 1. License and Terms of Use: if (datasetVersion.getTermsOfUseAndAccess() != null) { if (!TermsOfUseAndAccess.License.CC0.equals(datasetVersion.getTermsOfUseAndAccess().getLicense()) - && !(datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null - || datasetVersion.getTermsOfUseAndAccess().getTermsOfUse().equals(""))) { + && !(datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null + || datasetVersion.getTermsOfUseAndAccess().getTermsOfUse().equals(""))) { logger.fine("Download popup required because of license or terms of use."); return true; } @@ -1382,45 +1380,45 @@ public static File inputStreamToFile(InputStream inputStream) throws IOException } } - /* - * This method tells you if thumbnail generation is *supported* - * on this type of file. i.e., if true, it does not guarantee that a thumbnail - * can/will be generated; but it means that we can try. + /* + * This method tells you if thumbnail generation is *supported* + * on this type of file. i.e., if true, it does not guarantee that a thumbnail + * can/will be generated; but it means that we can try. */ public static boolean isThumbnailSupported (DataFile file) { if (file == null) { return false; } - + if (file.isHarvested() || "".equals(file.getStorageIdentifier())) { return false; } - + String contentType = file.getContentType(); - + // Some browsers (Chrome?) seem to identify FITS files as mime // type "image/fits" on upload; this is both incorrect (the official // mime type for FITS is "application/fits", and problematic: then - // the file is identified as an image, and the page will attempt to + // the file is identified as an image, and the page will attempt to // generate a preview - which of course is going to fail... if (MIME_TYPE_FITSIMAGE.equalsIgnoreCase(contentType)) { return false; } // besides most image/* types, we can generate thumbnails for // pdf and "world map" files: - - return (contentType != null && - (contentType.startsWith("image/") || + + return (contentType != null && + (contentType.startsWith("image/") || contentType.equalsIgnoreCase("application/pdf") || (file.isTabularData() && file.hasGeospatialTag()) || contentType.equalsIgnoreCase(MIME_TYPE_GEO_SHAPE))); } - - - /* - * The method below appears to be unnecessary; + + + /* + * The method below appears to be unnecessary; * it duplicates the method generateImageThumbnailFromFileAsBase64() from ImageThumbConverter; - * plus it creates an unnecessary temp file copy of the source file. + * plus it creates an unnecessary temp file copy of the source file. public static String rescaleImage(File file) throws IOException { if (file == null) { logger.info("file was null!!"); @@ -1442,7 +1440,7 @@ public static String rescaleImage(File file) throws IOException { return ImageThumbConverter.getImageAsBase64FromFile(resizedFile); } */ - + public static DatasetThumbnail getThumbnail(DataFile file) { String imageSourceBase64 = ImageThumbConverter.getImageThumbnailAsBase64(file, ImageThumbConverter.DEFAULT_THUMBNAIL_SIZE); @@ -1450,7 +1448,7 @@ public static DatasetThumbnail getThumbnail(DataFile file) { return defaultDatasetThumbnail; } - + public static boolean isPackageFile(DataFile dataFile) { return DataFileServiceBean.MIME_TYPE_PACKAGE_FILE.equalsIgnoreCase(dataFile.getContentType()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index bb125aafa52..fd059640780 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -24,8 +24,6 @@ import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; - -import org.apache.commons.io.IOUtils; import org.passay.CharacterRule; import org.apache.commons.io.IOUtils;