Skip to content

Commit

Permalink
turn down logging #9601
Browse files Browse the repository at this point in the history
  • Loading branch information
@pdurbin
pdurbin committed May 25, 2023
1 parent 18076ed commit 5a0bf24
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 18 deletions.
28 changes: 12 additions & 16 deletions src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1186,8 +1186,7 @@ public boolean fileMetadataExtractable(DataFile dataFile) {

// Inspired by fileMetadataExtractable, above
public boolean fileMetadataExtractableFromNetcdf(DataFile dataFile, Path tempLocationPath) {
logger.fine("fileMetadataExtractableFromNetcdf dataFileIn: " + dataFile + ". tempLocationPath: " + tempLocationPath);
logger.info("fileMetadataExtractableFromNetcdf dataFileIn: " + dataFile + ". tempLocationPath: " + tempLocationPath + ". contentType: " + dataFile.getContentType());
logger.fine("fileMetadataExtractableFromNetcdf dataFileIn: " + dataFile + ". tempLocationPath: " + tempLocationPath + ". contentType: " + dataFile.getContentType());
if (dataFile.getContentType() != null
&& (dataFile.getContentType().equals(FileUtil.MIME_TYPE_NETCDF)
|| dataFile.getContentType().equals(FileUtil.MIME_TYPE_XNETCDF)
Expand Down Expand Up @@ -1267,10 +1266,10 @@ public boolean extractMetadataFromNetcdf(String tempFileLocation, DataFile dataF

String dataFileLocation = null;
if (tempFileLocation != null) {
logger.info("tempFileLocation is non null. Setting dataFileLocation to " + tempFileLocation);
logger.fine("tempFileLocation is non null. Setting dataFileLocation to " + tempFileLocation);
dataFileLocation = tempFileLocation;
} else {
logger.info("tempFileLocation is null. Perhaps the file is alrady on disk or S3 direct upload is enabled.");
logger.fine("tempFileLocation is null. Perhaps the file is alrady on disk or S3 direct upload is enabled.");
File tempFile = null;
File localFile;
StorageIO<DataFile> storageIO;
Expand All @@ -1280,11 +1279,11 @@ public boolean extractMetadataFromNetcdf(String tempFileLocation, DataFile dataF
if (storageIO.isLocalFile()) {
localFile = storageIO.getFileSystemPath().toFile();
dataFileLocation = localFile.getAbsolutePath();
logger.info("extractMetadataFromNetcdf: file is local. Path: " + dataFileLocation);
logger.fine("extractMetadataFromNetcdf: file is local. Path: " + dataFileLocation);
} else {
Optional<Boolean> allow = JvmSettings.GEO_EXTRACT_S3_DIRECT_UPLOAD.lookupOptional(Boolean.class);
if (!(allow.isPresent() && allow.get())) {
logger.info("extractMetadataFromNetcdf: skipping because of config is set to not slow down S3 remote upload.");
logger.fine("extractMetadataFromNetcdf: skipping because of config is set to not slow down S3 remote upload.");
return false;
}
// Need to create a temporary local file:
Expand All @@ -1293,7 +1292,7 @@ public boolean extractMetadataFromNetcdf(String tempFileLocation, DataFile dataF
tempFileChannel.transferFrom(targetFileChannel, 0, storageIO.getSize());
}
dataFileLocation = tempFile.getAbsolutePath();
logger.info("extractMetadataFromNetcdf: file is on S3. Downloaded and saved to temp path: " + dataFileLocation);
logger.fine("extractMetadataFromNetcdf: file is on S3. Downloaded and saved to temp path: " + dataFileLocation);
}
} catch (IOException ex) {
logger.info("extractMetadataFromNetcdf, could not use storageIO for data file id " + dataFile.getId() + ". Exception: " + ex);
Expand All @@ -1309,7 +1308,7 @@ public boolean extractMetadataFromNetcdf(String tempFileLocation, DataFile dataF
// Locate metadata extraction plugin for the file format by looking
// it up with the Ingest Service Provider Registry:
NetcdfFileMetadataExtractor extractorPlugin = new NetcdfFileMetadataExtractor();
logger.info("creating file from " + dataFileLocation);
logger.fine("creating file from " + dataFileLocation);
File file = new File(dataFileLocation);
FileMetadataIngest extractedMetadata = extractorPlugin.ingestFile(file);
Map<String, Set<String>> extractedMetadataMap = extractedMetadata.getMetadataMap();
Expand Down Expand Up @@ -1347,11 +1346,11 @@ public boolean extractMetadataNcml(DataFile dataFile, Path tempLocationPath) {
InputStream inputStream = null;
String dataFileLocation = null;
if (tempLocationPath != null) {
logger.info("extractMetadataNcml: tempLocationPath is non null. Setting dataFileLocation to " + tempLocationPath);
logger.fine("extractMetadataNcml: tempLocationPath is non null. Setting dataFileLocation to " + tempLocationPath);
// This file was just uploaded and hasn't been saved to S3 or local storage.
dataFileLocation = tempLocationPath.toString();
} else {
logger.info("extractMetadataNcml: tempLocationPath null. Calling getExistingFile for dataFileLocation.");
logger.fine("extractMetadataNcml: tempLocationPath null. Calling getExistingFile for dataFileLocation.");
dataFileLocation = getExistingFile(dataFile, dataFileLocation);
}
if (dataFileLocation != null) {
Expand Down Expand Up @@ -1423,21 +1422,18 @@ private String getExistingFile(DataFile dataFile, String dataFileLocation) {
if (storageIO.isLocalFile()) {
localFile = storageIO.getFileSystemPath().toFile();
dataFileLocation = localFile.getAbsolutePath();
logger.fine("extractMetadataNcml: file is local. Path: " + dataFileLocation);
logger.info("getExistingFile: file is local. Path: " + dataFileLocation);
logger.fine("getExistingFile: file is local. Path: " + dataFileLocation);
} else {
// Need to create a temporary local file:
tempFile = File.createTempFile("tempFileExtractMetadataNcml", ".tmp");
try ( ReadableByteChannel targetFileChannel = (ReadableByteChannel) storageIO.getReadChannel(); FileChannel tempFileChannel = new FileOutputStream(tempFile).getChannel();) {
tempFileChannel.transferFrom(targetFileChannel, 0, storageIO.getSize());
}
dataFileLocation = tempFile.getAbsolutePath();
logger.fine("extractMetadataNcml: file is on S3. Downloaded and saved to temp path: " + dataFileLocation);
logger.info("getExistingFile: file is on S3. Downloaded and saved to temp path: " + dataFileLocation);
logger.fine("getExistingFile: file is on S3. Downloaded and saved to temp path: " + dataFileLocation);
}
} catch (IOException ex) {
logger.info("While attempting to extract NcML, could not use storageIO for data file id " + dataFile.getId() + ". Exception: " + ex);
logger.info("getExistingFile: While attempting to extract NcML, could not use storageIO for data file id " + dataFile.getId() + ". Exception: " + ex);
logger.fine("getExistingFile: While attempting to extract NcML, could not use storageIO for data file id " + dataFile.getId() + ". Exception: " + ex);
}
return dataFileLocation;
}
Expand Down
4 changes: 2 additions & 2 deletions .../dataverse/ingest/metadataextraction/impl/plugins/netcdf/NetcdfFileMetadataExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public FileMetadataIngest ingestFile(File file) throws IOException {
String northLatitudeFinal = geoFields.get(NORTH_LATITUDE);
String southLatitudeFinal = geoFields.get(SOUTH_LATITUDE);

logger.info(getLineStringsUrl(westLongitudeFinal, southLatitudeFinal, eastLongitudeFinal, northLatitudeFinal));
logger.fine(getLineStringsUrl(westLongitudeFinal, southLatitudeFinal, eastLongitudeFinal, northLatitudeFinal));

Map<String, Set<String>> metadataMap = new HashMap<>();
metadataMap.put(WEST_LONGITUDE, new HashSet<>());
Expand Down Expand Up @@ -102,7 +102,7 @@ private Map<String, String> parseGeospatial(NetcdfFile netcdfFile) {
geoFields.put(DatasetFieldConstant.northLatitude, getValue(northLatitude));
geoFields.put(DatasetFieldConstant.southLatitude, getValue(southLatitude));

logger.info(getLineStringsUrl(
logger.fine(getLineStringsUrl(
geoFields.get(DatasetFieldConstant.westLongitude),
geoFields.get(DatasetFieldConstant.southLatitude),
geoFields.get(DatasetFieldConstant.eastLongitude),
Expand Down

0 comments on commit 5a0bf24

Please sign in to comment.