From 5cd7cbe566bef2b2f5061a60bd92c7d1832587a8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 27 Apr 2023 10:53:24 -0400 Subject: [PATCH 01/12] avoid npe if no session seen once at QDR --- .../iq/dataverse/util/SessionUtil.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java index effa1980d70..d2fe8a4b0f9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java @@ -18,15 +18,17 @@ public class SessionUtil { */ public static void changeSessionId(HttpServletRequest h) { HttpSession session = h.getSession(false); - HashMap sessionAttributes = new HashMap(); - for(Enumeration e = session.getAttributeNames();e.hasMoreElements();) { - String name = e.nextElement(); - sessionAttributes.put(name, session.getAttribute(name)); - } - h.getSession().invalidate(); - session = h.getSession(true); - for(Entry entry: sessionAttributes.entrySet()) { - session.setAttribute(entry.getKey(), entry.getValue()); + if (session != null) { + HashMap sessionAttributes = new HashMap(); + for (Enumeration e = session.getAttributeNames(); e.hasMoreElements();) { + String name = e.nextElement(); + sessionAttributes.put(name, session.getAttribute(name)); + } + session.invalidate(); + session = h.getSession(true); + for (Entry entry : sessionAttributes.entrySet()) { + session.setAttribute(entry.getKey(), entry.getValue()); + } } } } From 229e1808d34df48a05df62d5ba76eb3cb687ac61 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 27 Apr 2023 11:05:09 -0400 Subject: [PATCH 02/12] always create session --- .../iq/dataverse/util/SessionUtil.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java index d2fe8a4b0f9..33e5296955d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java @@ -10,25 +10,30 @@ public class SessionUtil { /** - * Changes the session id (jsessionId) - for use when the session's authority increases (i.e. at login) - * Servlet 3.1 Note: This method is needed while using Servlets 2.0. 3.1 has a HttpServletRequest.chageSessionId(); method that can be used instead. - * - * @param h the current HttpServletRequest - * e.g. for pages you can get this from (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + * Changes the session id (jsessionId) - for use when the session's authority + * increases (i.e. at login) Servlet 3.1 Note: This method is needed while using + * Servlets 2.0. 3.1 has a HttpServletRequest.chageSessionId(); method that can + * be used instead. + * + * @param h + * the current HttpServletRequest e.g. for pages you can get this + * from (HttpServletRequest) + * FacesContext.getCurrentInstance().getExternalContext().getRequest(); */ public static void changeSessionId(HttpServletRequest h) { HttpSession session = h.getSession(false); + HashMap sessionAttributes = new HashMap(); if (session != null) { - HashMap sessionAttributes = new HashMap(); for (Enumeration e = session.getAttributeNames(); e.hasMoreElements();) { String name = e.nextElement(); sessionAttributes.put(name, session.getAttribute(name)); } session.invalidate(); - session = h.getSession(true); - for (Entry entry : sessionAttributes.entrySet()) { - session.setAttribute(entry.getKey(), entry.getValue()); - } } + session = h.getSession(true); + for (Entry entry : sessionAttributes.entrySet()) { + session.setAttribute(entry.getKey(), entry.getValue()); + } + } } From 2ecca8b3cfac7b58205408b818a4ccb8662a46ce Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Jul 2024 14:54:59 -0400 Subject: [PATCH 03/12] comment only sync with v6.3-dev --- .../iq/dataverse/ingest/IngestMessageBean.java | 2 +- .../harvard/iq/dataverse/util/SessionUtil.java | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java index f56fe608a52..7322338b71e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java @@ -75,7 +75,7 @@ public void onMessage(Message message) { ObjectMessage om = (ObjectMessage) message; ingestMessage = (IngestMessage) om.getObject(); - // if the lock was removed while an ingest was queued, ratake the lock + // if the lock was removed while an ingest was queued, retake the lock // The "if" is the first thing that addDatasetLock method does. // It has some complexity and would result in the code duplication if repeated here. // If that check would be removed from the addDatasetLock method in the future without diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java index 33e5296955d..c4ad349448d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SessionUtil.java @@ -10,15 +10,11 @@ public class SessionUtil { /** - * Changes the session id (jsessionId) - for use when the session's authority - * increases (i.e. at login) Servlet 3.1 Note: This method is needed while using - * Servlets 2.0. 3.1 has a HttpServletRequest.chageSessionId(); method that can - * be used instead. - * - * @param h - * the current HttpServletRequest e.g. for pages you can get this - * from (HttpServletRequest) - * FacesContext.getCurrentInstance().getExternalContext().getRequest(); + * Changes the session id (jsessionId) - for use when the session's authority increases (i.e. at login) + * Servlet 3.1 Note: This method is needed while using Servlets 2.0. 3.1 has a HttpServletRequest.chageSessionId(); method that can be used instead. + * + * @param h the current HttpServletRequest + * e.g. for pages you can get this from (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); */ public static void changeSessionId(HttpServletRequest h) { HttpSession session = h.getSession(false); @@ -34,6 +30,5 @@ public static void changeSessionId(HttpServletRequest h) { for (Entry entry : sessionAttributes.entrySet()) { session.setAttribute(entry.getKey(), entry.getValue()); } - } } From bbae0024da46da0a35e43e214150692e1789cc07 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Jul 2024 14:55:58 -0400 Subject: [PATCH 04/12] checking time for main finddeep call --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 4ee51cea2a7..e7b6c7ee054 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2004,8 +2004,11 @@ private String init(boolean initFull) { // ... And now the "real" working version lookup: if (versionId != null) { + logger.info("Start time: " + System.currentTimeMillis()); this.workingVersion = datasetVersionService.findDeep(versionId); + logger.info("End time: " + System.currentTimeMillis()); dataset = workingVersion.getDataset(); + logger.info("Dataset time: " + System.currentTimeMillis()); } if (workingVersion == null) { From ecc9fca2667997056c8c12b1ae60d1cce1401756 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 17 Jul 2024 16:33:43 -0400 Subject: [PATCH 05/12] reindex batches of 20 files instead of all at once --- .../search/SolrIndexServiceBean.java | 165 +++++++++--------- 1 file changed, 87 insertions(+), 78 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 460744d78e0..97042782855 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -34,7 +34,7 @@ public class SolrIndexServiceBean { private static final Logger logger = Logger.getLogger(SolrIndexServiceBean.class.getCanonicalName()); - + @EJB DvObjectServiceBean dvObjectService; @EJB @@ -54,27 +54,29 @@ public class SolrIndexServiceBean { public static String messageString = "message"; /** - * @deprecated Now that MyData has shipped in 4.1 we have no plans to change - * the unpublishedDataRelatedToMeModeEnabled boolean to false. We should - * probably remove the boolean altogether to simplify the code. + * @deprecated Now that MyData has shipped in 4.1 we have no plans to change the + * unpublishedDataRelatedToMeModeEnabled boolean to false. We should + * probably remove the boolean altogether to simplify the code. * - * This non-default mode changes the behavior of the "Data Related To Me" - * feature to be more like "**Unpublished** Data Related to Me" after you - * have changed this boolean to true and run "index all". + * This non-default mode changes the behavior of the "Data Related + * To Me" feature to be more like "**Unpublished** Data Related to + * Me" after you have changed this boolean to true and run "index + * all". * - * The "Data Related to Me" feature relies on *always* indexing permissions - * regardless of if the DvObject is published or not. + * The "Data Related to Me" feature relies on *always* indexing + * permissions regardless of if the DvObject is published or not. * - * In "Unpublished Data Related to Me" mode, we first check if the DvObject - * is published. If it's published, we set the search permissions to *only* - * contain "group_public", which is quick and cheap to do. If the DvObject - * in question is *not* public, we perform the expensive operation of - * rooting around in the system to determine who should be able to - * "discover" the unpublished version of DvObject. By default this mode is - * *not* enabled. If you want to enable it, change the boolean to true and - * run "index all". + * In "Unpublished Data Related to Me" mode, we first check if the + * DvObject is published. If it's published, we set the search + * permissions to *only* contain "group_public", which is quick and + * cheap to do. If the DvObject in question is *not* public, we + * perform the expensive operation of rooting around in the system + * to determine who should be able to "discover" the unpublished + * version of DvObject. By default this mode is *not* enabled. If + * you want to enable it, change the boolean to true and run "index + * all". * - * See also https://github.com/IQSS/dataverse/issues/50 + * See also https://github.com/IQSS/dataverse/issues/50 */ @Deprecated private final boolean unpublishedDataRelatedToMeModeEnabled = true; @@ -116,8 +118,8 @@ private List determineSolrDocsForFilesFromDataset(Map.Entry perms = new ArrayList<>(); @@ -149,7 +151,7 @@ private List constructDatasetSolrDocs(Dataset dataset) { return solrDocs; } -// private List constructDatafileSolrDocs(DataFile dataFile) { + // private List constructDatafileSolrDocs(DataFile dataFile) { private List constructDatafileSolrDocs(DataFile dataFile, Map> permStringByDatasetVersion) { List datafileSolrDocs = new ArrayList<>(); Map desiredCards = searchPermissionsService.getDesiredCards(dataFile.getOwner()); @@ -177,8 +179,8 @@ private List constructDatafileSolrDocs(DataFile dataFile, Map constructDatafileSolrDocsFromDataset(Dataset datas } else { perms = searchPermissionsService.findDatasetVersionPerms(datasetVersionFileIsAttachedTo); } - + for (FileMetadata fileMetadata : datasetVersionFileIsAttachedTo.getFileMetadatas()) { Long fileId = fileMetadata.getDataFile().getId(); String solrIdStart = IndexServiceBean.solrDocIdentifierFile + fileId; String solrIdEnd = getDatasetOrDataFileSolrEnding(datasetVersionFileIsAttachedTo.getVersionState()); String solrId = solrIdStart + solrIdEnd; List ftperms = new ArrayList<>(); - if(fileMetadata.getDataFile().isRestricted()) { + if (fileMetadata.getDataFile().isRestricted()) { ftperms = searchPermissionsService.findDataFilePermsforDatasetVersion(fileMetadata.getDataFile(), datasetVersionFileIsAttachedTo); } DvObjectSolrDoc dataFileSolrDoc = new DvObjectSolrDoc(fileId.toString(), solrId, datasetVersionFileIsAttachedTo.getId(), fileMetadata.getLabel(), perms, ftperms); @@ -311,8 +313,8 @@ public IndexResponse indexAllPermissions() { try { persistToSolr(docs); /** - * @todo Do we need a separate permissionIndexTime timestamp? - * Probably. Update it here. + * @todo Do we need a separate permissionIndexTime timestamp? Probably. Update + * it here. */ for (DvObject dvObject : all) { dvObjectService.updatePermissionIndexTime(dvObject); @@ -368,14 +370,14 @@ private void persistToSolr(Collection docs) throws SolrServer public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) { DvObject definitionPoint = dvObjectService.findDvObject(definitionPointId); - if ( definitionPoint == null ) { + if (definitionPoint == null) { logger.log(Level.WARNING, "Cannot find a DvOpbject with id of {0}", definitionPointId); return null; } else { return indexPermissionsOnSelfAndChildren(definitionPoint); } } - + /** * We use the database to determine direct children since there is no * inheritance @@ -384,8 +386,8 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) List dvObjectsToReindexPermissionsFor = new ArrayList<>(); List filesToReindexAsBatch = new ArrayList<>(); /** - * @todo Re-indexing the definition point itself seems to be necessary - * for revoke but not necessarily grant. + * @todo Re-indexing the definition point itself seems to be necessary for + * revoke but not necessarily grant. */ // We don't create a Solr "primary/content" doc for the root dataverse @@ -416,8 +418,8 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) /** * @todo Error handling? What to do with response? * - * @todo Should update timestamps, probably, even thought these are - * files, see https://github.com/IQSS/dataverse/issues/2421 + * @todo Should update timestamps, probably, even thought these are files, see + * https://github.com/IQSS/dataverse/issues/2421 */ String response = reindexFilesInBatches(filesToReindexAsBatch); @@ -427,56 +429,64 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) */ IndexResponse indexResponse = indexPermissionsForOneDvObject(dvObject); } - + return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint - + ": " + dvObjectsToReindexPermissionsFor.size() - ); + + ": " + dvObjectsToReindexPermissionsFor.size()); } private String reindexFilesInBatches(List filesToReindexPermissionsFor) { List docs = new ArrayList<>(); Map> byParentId = new HashMap<>(); Map> permStringByDatasetVersion = new HashMap<>(); - for (DataFile file : filesToReindexPermissionsFor) { - Dataset dataset = (Dataset) file.getOwner(); - Map desiredCards = searchPermissionsService.getDesiredCards(dataset); - for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataset)) { - boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState()); - if (cardShouldExist) { - List cachedPermission = permStringByDatasetVersion.get(datasetVersionFileIsAttachedTo.getId()); - if (cachedPermission == null) { - logger.fine("no cached permission! Looking it up..."); - List fileSolrDocs = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion); - for (DvObjectSolrDoc fileSolrDoc : fileSolrDocs) { - Long datasetVersionId = fileSolrDoc.getDatasetVersionId(); - if (datasetVersionId != null) { - permStringByDatasetVersion.put(datasetVersionId, fileSolrDoc.getPermissions()); + int i = 0; + try { + for (DataFile file : filesToReindexPermissionsFor) { + Dataset dataset = (Dataset) file.getOwner(); + Map desiredCards = searchPermissionsService.getDesiredCards(dataset); + for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataset)) { + boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState()); + if (cardShouldExist) { + List cachedPermission = permStringByDatasetVersion.get(datasetVersionFileIsAttachedTo.getId()); + if (cachedPermission == null) { + logger.fine("no cached permission! Looking it up..."); + List fileSolrDocs = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion); + for (DvObjectSolrDoc fileSolrDoc : fileSolrDocs) { + Long datasetVersionId = fileSolrDoc.getDatasetVersionId(); + if (datasetVersionId != null) { + permStringByDatasetVersion.put(datasetVersionId, fileSolrDoc.getPermissions()); + SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc); + docs.add(solrDoc); + i++; + } + } + } else { + logger.fine("cached permission is " + cachedPermission); + List fileSolrDocsBasedOnCachedPermissions = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion); + for (DvObjectSolrDoc fileSolrDoc : fileSolrDocsBasedOnCachedPermissions) { SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc); docs.add(solrDoc); + i++; } } - } else { - logger.fine("cached permission is " + cachedPermission); - List fileSolrDocsBasedOnCachedPermissions = constructDatafileSolrDocs((DataFile) file, permStringByDatasetVersion); - for (DvObjectSolrDoc fileSolrDoc : fileSolrDocsBasedOnCachedPermissions) { - SolrInputDocument solrDoc = SearchUtil.createSolrDoc(fileSolrDoc); - docs.add(solrDoc); + if (i % 20 == 0) { + persistToSolr(docs); + docs = new ArrayList<>(); + i = 0; } } } + Long parent = file.getOwner().getId(); + List existingList = byParentId.get(parent); + if (existingList == null) { + List empty = new ArrayList<>(); + byParentId.put(parent, empty); + } else { + List updatedList = existingList; + updatedList.add(file.getId()); + byParentId.put(parent, updatedList); + } } - Long parent = file.getOwner().getId(); - List existingList = byParentId.get(parent); - if (existingList == null) { - List empty = new ArrayList<>(); - byParentId.put(parent, empty); - } else { - List updatedList = existingList; - updatedList.add(file.getId()); - byParentId.put(parent, updatedList); - } - } - try { + persistToSolr(docs); return " " + filesToReindexPermissionsFor.size() + " files indexed across " + docs.size() + " Solr documents "; } catch (SolrServerException | IOException ex) { @@ -526,27 +536,26 @@ public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServer /** * * - * @return A list of dvobject ids that should have their permissions - * re-indexed because Solr was down when a permission was added. The permission - * should be added to Solr. The id of the permission contains the type of - * DvObject and the primary key of the dvObject. - * DvObjects of type DataFile are currently skipped because their index - * time isn't stored in the database, since they are indexed along - * with their parent dataset (this may change). + * @return A list of dvobject ids that should have their permissions re-indexed + * because Solr was down when a permission was added. The permission + * should be added to Solr. The id of the permission contains the type + * of DvObject and the primary key of the dvObject. DvObjects of type + * DataFile are currently skipped because their index time isn't stored + * in the database, since they are indexed along with their parent + * dataset (this may change). */ public List findPermissionsInDatabaseButStaleInOrMissingFromSolr() { List indexingRequired = new ArrayList<>(); long rootDvId = dataverseService.findRootDataverse().getId(); List missingDataversePermissionIds = dataverseService.findIdStalePermission(); List missingDatasetPermissionIds = datasetService.findIdStalePermission(); - for (Long id : missingDataversePermissionIds) { + for (Long id : missingDataversePermissionIds) { if (!id.equals(rootDvId)) { - indexingRequired.add(id); + indexingRequired.add(id); } } indexingRequired.addAll(missingDatasetPermissionIds); return indexingRequired; } - } From f8138505717595dd6c72521a4721bfd6968b4467 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 17 Jul 2024 17:39:10 -0400 Subject: [PATCH 06/12] Also only keep 100 files in list at a time --- .../iq/dataverse/search/SolrIndexServiceBean.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 97042782855..6aecaa38f3a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -392,6 +392,7 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) // We don't create a Solr "primary/content" doc for the root dataverse // so don't create a Solr "permission" doc either. + int i = 0; if (definitionPoint.isInstanceofDataverse()) { Dataverse selfDataverse = (Dataverse) definitionPoint; if (!selfDataverse.equals(dataverseService.findRootDataverse())) { @@ -402,6 +403,11 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) dvObjectsToReindexPermissionsFor.add(dataset); for (DataFile datafile : filesToReIndexPermissionsFor(dataset)) { filesToReindexAsBatch.add(datafile); + i++; + if (i % 100 == 0) { + reindexFilesInBatches(filesToReindexAsBatch); + filesToReindexAsBatch.clear(); + } } } } else if (definitionPoint.isInstanceofDataset()) { @@ -410,6 +416,11 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) Dataset dataset = (Dataset) definitionPoint; for (DataFile datafile : filesToReIndexPermissionsFor(dataset)) { filesToReindexAsBatch.add(datafile); + i++; + if (i % 100 == 0) { + reindexFilesInBatches(filesToReindexAsBatch); + filesToReindexAsBatch.clear(); + } } } else { dvObjectsToReindexPermissionsFor.add(definitionPoint); @@ -422,7 +433,7 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) * https://github.com/IQSS/dataverse/issues/2421 */ String response = reindexFilesInBatches(filesToReindexAsBatch); - + logger.fine("Reindexed permissions for " + i + " files"); for (DvObject dvObject : dvObjectsToReindexPermissionsFor) { /** * @todo do something with this response @@ -471,7 +482,6 @@ private String reindexFilesInBatches(List filesToReindexPermissionsFor if (i % 20 == 0) { persistToSolr(docs); docs = new ArrayList<>(); - i = 0; } } } From 500bac50a3bb3cca8f18b6ad449dd7328f3bdce9 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 18 Jul 2024 10:09:25 -0400 Subject: [PATCH 07/12] tweak ror styling --- src/main/webapp/resources/css/structure.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index edf040cd94f..caa8d382aff 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -1288,6 +1288,11 @@ ul > li:not(.ui-selectonemenu-item) { #metadata_author img { vertical-align: text-top; margin-right:1px; + margin-left: 3px; +} + +#metadata_author img.ror { + vertical-align: middle; } #metadata_dsDescription ul > li { From d968d77ea073309cc35de87063c415f51f1ba6b7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 18 Jul 2024 16:44:19 -0400 Subject: [PATCH 08/12] Just do collections/datasets as you go Avoids keeping everything in memory, also helps in tracking progress as you can see the permissionindextime getting updated per dataset. --- .../search/SolrIndexServiceBean.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 6aecaa38f3a..c44d44901ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -383,7 +383,6 @@ public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) { * inheritance */ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) { - List dvObjectsToReindexPermissionsFor = new ArrayList<>(); List filesToReindexAsBatch = new ArrayList<>(); /** * @todo Re-indexing the definition point itself seems to be necessary for @@ -393,14 +392,17 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) // We don't create a Solr "primary/content" doc for the root dataverse // so don't create a Solr "permission" doc either. int i = 0; + int numObjects = 0; if (definitionPoint.isInstanceofDataverse()) { Dataverse selfDataverse = (Dataverse) definitionPoint; if (!selfDataverse.equals(dataverseService.findRootDataverse())) { - dvObjectsToReindexPermissionsFor.add(definitionPoint); + indexPermissionsForOneDvObject(definitionPoint); + numObjects++; } List directChildDatasetsOfDvDefPoint = datasetService.findByOwnerId(selfDataverse.getId()); for (Dataset dataset : directChildDatasetsOfDvDefPoint) { - dvObjectsToReindexPermissionsFor.add(dataset); + indexPermissionsForOneDvObject(dataset); + numObjects++; for (DataFile datafile : filesToReIndexPermissionsFor(dataset)) { filesToReindexAsBatch.add(datafile); i++; @@ -411,7 +413,8 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) } } } else if (definitionPoint.isInstanceofDataset()) { - dvObjectsToReindexPermissionsFor.add(definitionPoint); + indexPermissionsForOneDvObject(definitionPoint); + numObjects++; // index files Dataset dataset = (Dataset) definitionPoint; for (DataFile datafile : filesToReIndexPermissionsFor(dataset)) { @@ -423,7 +426,8 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) } } } else { - dvObjectsToReindexPermissionsFor.add(definitionPoint); + indexPermissionsForOneDvObject(definitionPoint); + numObjects++; } /** @@ -432,17 +436,10 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) * @todo Should update timestamps, probably, even thought these are files, see * https://github.com/IQSS/dataverse/issues/2421 */ - String response = reindexFilesInBatches(filesToReindexAsBatch); + reindexFilesInBatches(filesToReindexAsBatch); logger.fine("Reindexed permissions for " + i + " files"); - for (DvObject dvObject : dvObjectsToReindexPermissionsFor) { - /** - * @todo do something with this response - */ - IndexResponse indexResponse = indexPermissionsForOneDvObject(dvObject); - } - return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint - + ": " + dvObjectsToReindexPermissionsFor.size()); + + ": " + numObjects); } private String reindexFilesInBatches(List filesToReindexPermissionsFor) { From b8c2acea725f655c5b73cc821e016deb721508a7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Jul 2024 10:11:22 -0400 Subject: [PATCH 09/12] add logging --- .../harvard/iq/dataverse/search/SolrIndexServiceBean.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index c44d44901ac..edc4bbb9ccd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -410,7 +410,11 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) reindexFilesInBatches(filesToReindexAsBatch); filesToReindexAsBatch.clear(); } + if (i % 1000 == 0) { + logger.info("Progress: " +i + "files permissions reindexed"); + } } + logger.info("Progress : dataset " + dataset.getId() + " permissions reindexed"); } } else if (definitionPoint.isInstanceofDataset()) { indexPermissionsForOneDvObject(definitionPoint); @@ -437,7 +441,7 @@ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) * https://github.com/IQSS/dataverse/issues/2421 */ reindexFilesInBatches(filesToReindexAsBatch); - logger.fine("Reindexed permissions for " + i + " files"); + logger.info("Reindexed permissions for " + i + " files and " + numObjects + "datasets/collections"); return new IndexResponse("Number of dvObject permissions indexed for " + definitionPoint + ": " + numObjects); } From 0abcc79d67f1d9a3d752026161918c662e23c3ac Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 19 Jul 2024 11:47:54 -0400 Subject: [PATCH 10/12] fix solrconfig --- conf/solr/solrconfig.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/solr/solrconfig.xml b/conf/solr/solrconfig.xml index 8fc0a266f29..6fc34d2fad4 100644 --- a/conf/solr/solrconfig.xml +++ b/conf/solr/solrconfig.xml @@ -590,9 +590,9 @@ true 75 --> - +