Skip to content

Commit

Permalink
Add lucene.searchBlockSize and refactor getReadable #277
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrick-austin
patrick-austin committed Mar 9, 2022
1 parent 3770bbf commit 7bf55e9
Show file tree
Hide file tree
Showing 11 changed files with 137 additions and 118 deletions.
4 changes: 4 additions & 0 deletions src/main/config/run.properties.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ log.list = SESSION WRITE READ INFO
# Lucene
lucene.url = https://localhost:8181
lucene.populateBlockSize = 10000
# Recommend setting lucene.searchBlockSize equal to maxIdsInQuery, so that all Lucene results can be authorised at once
# If lucene.searchBlockSize > maxIdsInQuery, then multiple auth checks may be needed for a single search to Lucene
# The optimal value depends on how likely a user's auth request fails: larger values are more efficient when rejection is more likely
lucene.searchBlockSize = 1000
lucene.directory = ${HOME}/data/icat/lucene
lucene.backlogHandlerIntervalSeconds = 60
lucene.enqueuedRequestIntervalSeconds = 5
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/icatproject/core/entity/EntityBaseBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,15 @@
import org.icatproject.core.manager.EntityInfoHandler;
import org.icatproject.core.manager.EntityInfoHandler.Relationship;
import org.icatproject.core.manager.GateKeeper;
import org.icatproject.core.manager.HasEntityId;
import org.icatproject.core.manager.LuceneManager;
import org.icatproject.core.parser.IncludeClause.Step;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@SuppressWarnings("serial")
@MappedSuperclass
public abstract class EntityBaseBean implements Serializable {
public abstract class EntityBaseBean implements HasEntityId, Serializable {

private static final EntityInfoHandler eiHandler = EntityInfoHandler.getInstance();

Expand Down
62 changes: 36 additions & 26 deletions src/main/java/org/icatproject/core/manager/EntityBeanManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public enum PersistMode {
private boolean luceneActive;

private int maxEntities;
private int maxIdsInQuery;
private int luceneSearchBlockSize;

private long exportCacheSize;
private Set<String> rootUserNames;
Expand Down Expand Up @@ -782,28 +782,38 @@ private void exportTable(String beanName, Set<Long> ids, OutputStream output,
}
}

private void filterReadAccess(List<ScoredEntityBaseBean> results, List<ScoredEntityBaseBean> allResults,
private void filterReadAccess(List<ScoredEntityBaseBean> acceptedResults, List<ScoredEntityBaseBean> newResults,
int maxCount, String userId, EntityManager manager, Class<? extends EntityBaseBean> klass)
throws IcatException {

logger.debug("Got " + allResults.size() + " results from Lucene");
List<Long> allIds = new ArrayList<>();
allResults.forEach(r -> allIds.add(r.getEntityBaseBeanId()));
List<Long> allowedIds = gateKeeper.getReadableIds(userId, allIds, klass, manager);
for (ScoredEntityBaseBean sr : allResults) {
try {
if (allowedIds.contains(sr.getEntityBaseBeanId())) {
results.add(sr);
}
if (results.size() > maxEntities) {
throw new IcatException(IcatExceptionType.VALIDATION,
"attempt to return more than " + maxEntities + " entities");
}
if (results.size() == maxCount) {
break;
logger.debug("Got " + newResults.size() + " results from Lucene");
Set<Long> allowedIds = gateKeeper.getReadableIds(userId, newResults, klass.getSimpleName(), manager);
if (allowedIds == null) {
// A null result means there are no restrictions on the readable ids, so add as
// many newResults as we need to reach maxCount
int needed = maxCount - acceptedResults.size();
if (newResults.size() > needed) {
acceptedResults.addAll(newResults.subList(0, needed));
} else {
acceptedResults.addAll(newResults);
}
if (acceptedResults.size() > maxEntities) {
throw new IcatException(IcatExceptionType.VALIDATION,
"attempt to return more than " + maxEntities + " entities");
}
} else {
// Otherwise, add results in order until we reach maxCount
for (ScoredEntityBaseBean newResult : newResults) {
if (allowedIds.contains(newResult.getId())) {
acceptedResults.add(newResult);
if (acceptedResults.size() > maxEntities) {
throw new IcatException(IcatExceptionType.VALIDATION,
"attempt to return more than " + maxEntities + " entities");
}
if (acceptedResults.size() == maxCount) {
break;
}
}
} catch (IcatException e) {
// Nothing to do
}
}
}
Expand Down Expand Up @@ -1155,7 +1165,7 @@ void init() {
notificationRequests = propertyHandler.getNotificationRequests();
luceneActive = lucene.isActive();
maxEntities = propertyHandler.getMaxEntities();
maxIdsInQuery = propertyHandler.getMaxIdsInQuery();
luceneSearchBlockSize = propertyHandler.getLuceneSearchBlockSize();
exportCacheSize = propertyHandler.getImportCacheSize();
rootUserNames = propertyHandler.getRootUserNames();
key = propertyHandler.getKey();
Expand Down Expand Up @@ -1400,7 +1410,7 @@ public List<ScoredEntityBaseBean> luceneDatafiles(String userName, String user,
LuceneSearchResult last = null;
Long uid = null;
List<ScoredEntityBaseBean> allResults = Collections.emptyList();
int blockSize = maxIdsInQuery;
int blockSize = luceneSearchBlockSize;

do {
if (last == null) {
Expand All @@ -1421,7 +1431,7 @@ public List<ScoredEntityBaseBean> luceneDatafiles(String userName, String user,
try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) {
gen.write("userName", userName);
if (results.size() > 0) {
gen.write("entityId", results.get(0).getEntityBaseBeanId());
gen.write("entityId", results.get(0).getId());
}
gen.writeEnd();
}
Expand All @@ -1439,7 +1449,7 @@ public List<ScoredEntityBaseBean> luceneDatasets(String userName, String user, S
LuceneSearchResult last = null;
Long uid = null;
List<ScoredEntityBaseBean> allResults = Collections.emptyList();
int blockSize = maxIdsInQuery;
int blockSize = luceneSearchBlockSize;

do {
if (last == null) {
Expand All @@ -1459,7 +1469,7 @@ public List<ScoredEntityBaseBean> luceneDatasets(String userName, String user, S
try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) {
gen.write("userName", userName);
if (results.size() > 0) {
gen.write("entityId", results.get(0).getEntityBaseBeanId());
gen.write("entityId", results.get(0).getId());
}
gen.writeEnd();
}
Expand All @@ -1486,7 +1496,7 @@ public List<ScoredEntityBaseBean> luceneInvestigations(String userName, String u
LuceneSearchResult last = null;
Long uid = null;
List<ScoredEntityBaseBean> allResults = Collections.emptyList();
int blockSize = maxIdsInQuery;
int blockSize = luceneSearchBlockSize;

do {
if (last == null) {
Expand All @@ -1506,7 +1516,7 @@ public List<ScoredEntityBaseBean> luceneInvestigations(String userName, String u
try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) {
gen.write("userName", userName);
if (results.size() > 0) {
gen.write("entityId", results.get(0).getEntityBaseBeanId());
gen.write("entityId", results.get(0).getId());
}
gen.writeEnd();
}
Expand Down
147 changes: 65 additions & 82 deletions src/main/java/org/icatproject/core/manager/GateKeeper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,113 +164,102 @@ public Set<String> getPublicTables() {
return publicTables;
}

public List<EntityBaseBean> getReadable(String userId, List<EntityBaseBean> beans, EntityManager manager) {

if (beans.size() == 0) {
return beans;
}

EntityBaseBean object = beans.get(0);

Class<? extends EntityBaseBean> objectClass = object.getClass();
String simpleName = objectClass.getSimpleName();
/**
* @param userId The user making the READ request
* @param simpleName The name of the requested entity type
* @param manager
* @return Returns a list of restrictions that apply to the requested entity
* type. If there are no restrictions, then returns null.
*/
private List<String> getRestrictions(String userId, String simpleName, EntityManager manager) {
if (rootUserNames.contains(userId)) {
logger.info("\"Root\" user " + userId + " is allowed READ to " + simpleName);
return beans;
return null;
}

TypedQuery<String> query = manager.createNamedQuery(Rule.INCLUDE_QUERY, String.class)
.setParameter("member", userId).setParameter("bean", simpleName);

List<String> restrictions = query.getResultList();
logger.debug("Got " + restrictions.size() + " authz queries for READ by " + userId + " to a "
+ objectClass.getSimpleName());
+ simpleName);

for (String restriction : restrictions) {
logger.debug("Query: " + restriction);
if (restriction == null) {
logger.info("Null restriction => READ permitted to " + simpleName);
return beans;
return null;
}
}

/*
* IDs are processed in batches to avoid Oracle error: ORA-01795:
* maximum number of expressions in a list is 1000
*/
return restrictions;
}

List<String> idLists = new ArrayList<>();
StringBuilder sb = null;
/**
* Returns a sub list of the passed entities that the user has READ access to.
*
* @param userId The user making the READ request
* @param beans The entities the user wants to READ
* @param manager
* @return A list of entities the user has read access to
*/
public List<EntityBaseBean> getReadable(String userId, List<EntityBaseBean> beans, EntityManager manager) {

int i = 0;
for (EntityBaseBean bean : beans) {
if (i == 0) {
sb = new StringBuilder();
sb.append(bean.getId());
i = 1;
} else {
sb.append("," + bean.getId());
i++;
}
if (i == maxIdsInQuery) {
i = 0;
idLists.add(sb.toString());
sb = null;
}
}
if (sb != null) {
idLists.add(sb.toString());
if (beans.size() == 0) {
return beans;
}
EntityBaseBean object = beans.get(0);
Class<? extends EntityBaseBean> objectClass = object.getClass();
String simpleName = objectClass.getSimpleName();

logger.debug("Check readability of " + beans.size() + " beans has been divided into " + idLists.size()
+ " queries.");

Set<Long> ids = new HashSet<>();
for (String idList : idLists) {
for (String qString : restrictions) {
TypedQuery<Long> q = manager.createQuery(qString.replace(":pkids", idList), Long.class);
if (qString.contains(":user")) {
q.setParameter("user", userId);
}
ids.addAll(q.getResultList());
}
List<String> restrictions = getRestrictions(userId, simpleName, manager);
if (restrictions == null) {
return beans;
}

Set<Long> readableIds = getReadableIds(userId, beans, restrictions, manager);

List<EntityBaseBean> results = new ArrayList<>();
for (EntityBaseBean bean : beans) {
if (ids.contains(bean.getId())) {
if (readableIds.contains(bean.getId())) {
results.add(bean);
}
}
return results;
}

public List<Long> getReadableIds(String userId, List<Long> ids,
Class<? extends EntityBaseBean> objectClass, EntityManager manager) {
if (ids.size() == 0) {
return ids;
}
/**
* @param userId The user making the READ request
* @param entities The entities to check
* @param simpleName The name of the requested entity type
* @param manager
* @return Set of the ids that the user has read access to. If there are no
* restrictions, then returns null.
*/
public Set<Long> getReadableIds(String userId, List<? extends HasEntityId> entities, String simpleName,
EntityManager manager) {

String simpleName = objectClass.getSimpleName();
if (rootUserNames.contains(userId)) {
logger.info("\"Root\" user " + userId + " is allowed READ to " + simpleName);
return ids;
if (entities.size() == 0) {
return null;
}

TypedQuery<String> query = manager.createNamedQuery(Rule.INCLUDE_QUERY, String.class)
.setParameter("member", userId).setParameter("bean", simpleName);
List<String> restrictions = getRestrictions(userId, simpleName, manager);
if (restrictions == null) {
return null;
}

List<String> restrictions = query.getResultList();
logger.debug("Got " + restrictions.size() + " authz queries for READ by " + userId + " to a "
+ objectClass.getSimpleName());
return getReadableIds(userId, entities, restrictions, manager);
}

for (String restriction : restrictions) {
logger.debug("Query: " + restriction);
if (restriction == null) {
logger.info("Null restriction => READ permitted to " + simpleName);
return ids;
}
}
/**
* @param userId The user making the READ request
* @param entities The entities to check
* @param restrictions The restrictions applying to the entities
* @param manager
* @return Set of the ids that the user has read access to
*/
private Set<Long> getReadableIds(String userId, List<? extends HasEntityId> entities, List<String> restrictions,
EntityManager manager) {

/*
* IDs are processed in batches to avoid Oracle error: ORA-01795:
Expand All @@ -281,13 +270,13 @@ public List<Long> getReadableIds(String userId, List<Long> ids,
StringBuilder sb = null;

int i = 0;
for (Long id : ids) {
for (HasEntityId entity : entities) {
if (i == 0) {
sb = new StringBuilder();
sb.append(id);
sb.append(entity.getId());
i = 1;
} else {
sb.append("," + id);
sb.append("," + entity.getId());
i++;
}
if (i == maxIdsInQuery) {
Expand All @@ -300,7 +289,7 @@ public List<Long> getReadableIds(String userId, List<Long> ids,
idLists.add(sb.toString());
}

logger.debug("Check readability of " + ids.size() + " beans has been divided into " + idLists.size()
logger.debug("Check readability of " + entities.size() + " beans has been divided into " + idLists.size()
+ " queries.");

Set<Long> readableIds = new HashSet<>();
Expand All @@ -314,13 +303,7 @@ public List<Long> getReadableIds(String userId, List<Long> ids,
}
}

List<Long> results = new ArrayList<>();
for (Long id : ids) {
if (readableIds.contains(id)) {
results.add(id);
}
}
return results;
return readableIds;
}

public Set<String> getRootUserNames() {
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/icatproject/core/manager/HasEntityId.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package org.icatproject.core.manager;

/**
* Interface for objects representing entities that hold the entity id.
*/
public interface HasEntityId {
public Long getId();
}
Loading

0 comments on commit 7bf55e9

Please sign in to comment.