Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OAK-9447 #1761

Open
wants to merge 12 commits into
base: trunk
Choose a base branch
from
Open

OAK-9447 #1761

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
84fa64f
OAK-9447
Oct 4, 2024
435903a
Conflict with trunk resolved
Oct 4, 2024
50af9a3
Conflict resolved
Oct 4, 2024
afd034e
conflict resolved
Oct 4, 2024
a573920
Merge branch 'trunk' into trunk
rgambelli Oct 4, 2024
853b93d
Reordered imports like original classes and fixed oak-run about new
Oct 4, 2024
9840ccc
Reordered imports like original classes
Oct 4, 2024
0bc6b06
Merge branch 'trunk' into trunk
rgambelli Oct 6, 2024
10b1cfc
Upgraded com.mongodb and org.bson versions in Import-Package
Oct 8, 2024
bac8f9d
Restored original formatting
Oct 8, 2024
01c5b2d
Restored original import ordering
Oct 8, 2024
59576cb
Restored original import ordering
Oct 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion oak-benchmarks/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
Expand Down
3 changes: 1 addition & 2 deletions oak-examples/standalone/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>${mongo.driver.version}</version>
<artifactId>mongodb-driver-sync</artifactId>
<optional>true</optional>
</dependency>
<!-- Required for the persistent cache -->
Expand Down
2 changes: 1 addition & 1 deletion oak-it/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
<optional>true</optional>
<scope>test</scope>
</dependency>
Expand Down
2 changes: 1 addition & 1 deletion oak-jcr/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
7 changes: 2 additions & 5 deletions oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/cluster/NonLocalObservationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,18 @@
import static org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.DOCUMENT_NS;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;

import java.util.Date;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;

import javax.jcr.Node;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.observation.Event;
import javax.jcr.observation.EventIterator;
import javax.jcr.observation.EventListener;
import javax.jcr.observation.ObservationManager;

import org.apache.jackrabbit.api.observation.JackrabbitEvent;
import org.apache.jackrabbit.oak.commons.FixturesHelper;
import org.apache.jackrabbit.oak.fixture.DocumentMongoFixture;
Expand All @@ -47,8 +44,8 @@
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.client.MongoClient;

import com.mongodb.MongoClient;

/**
* Test for external events from another cluster node.
Expand Down Expand Up @@ -108,7 +105,7 @@ public void dispose(NodeStore nodeStore) {
nodeStores.remove(nodeStore);
if (nodeStores.size() == 0) {
try (MongoClient c = createClient()) {
c.dropDatabase(dbName);
c.getDatabase(dbName).drop();
} catch (Exception e) {
log.error("dispose: Can't close Mongo", e);
}
Expand Down
4 changes: 0 additions & 4 deletions oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/random/RandomOpCompare.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,21 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.Closeable;
import java.io.IOException;
import java.util.Random;

import javax.jcr.Node;
import javax.jcr.Repository;
import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.SimpleCredentials;

import org.apache.jackrabbit.oak.NodeStoreFixtures;
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import org.apache.jackrabbit.oak.jcr.Jcr;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.spi.state.NodeStore;

import com.mongodb.DB;

/**
* A randomized test that writes to two repositories (using different storage
Expand Down
2 changes: 1 addition & 1 deletion oak-lucene/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
6 changes: 3 additions & 3 deletions oak-parent/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
<segment.db>SegmentMK</segment.db>
<lucene.version>4.7.2</lucene.version>
<solr.version>8.11.1</solr.version>
<mongo.driver.version>3.12.11</mongo.driver.version>
<mongo.driver.version>5.2.0</mongo.driver.version>
<slf4j.api.version>1.7.36</slf4j.api.version>
<slf4j.version>1.7.36</slf4j.version> <!-- sync with logback version -->
<logback.version>1.2.13</logback.version>
Expand Down Expand Up @@ -558,9 +558,9 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
<version>${mongo.driver.version}</version>
</dependency>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion oak-pojosr/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
<optional>true</optional>
</dependency>
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion oak-run-commons/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<artifactId>mongodb-driver-sync</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
Expand Down
21 changes: 9 additions & 12 deletions oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/CompositeStoreFixture.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@

package org.apache.jackrabbit.oak.fixture;

import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentNodeStoreBuilder.newMongoDocumentNodeStoreBuilder;
import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.composite.CompositeNodeStore;
import org.apache.jackrabbit.oak.composite.InitialContentMigrator;
Expand All @@ -37,17 +44,7 @@
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import com.mongodb.MongoClientURI;

import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentNodeStoreBuilder.newMongoDocumentNodeStoreBuilder;
import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import com.mongodb.ConnectionString;

abstract class CompositeStoreFixture extends OakFixture {

Expand Down Expand Up @@ -109,7 +106,7 @@ static OakFixture newCompositeMongoFixture(String name, String uri, boolean drop
boolean throttlingEnabled) {
return new CompositeStoreFixture(name) {

private String database = new MongoClientURI(uri).getDatabase();
private String database = new ConnectionString(uri).getDatabase();
private DocumentNodeStore ns;

@Override
Expand Down
73 changes: 37 additions & 36 deletions .../main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,33 @@

package org.apache.jackrabbit.oak.index.indexer.document;

import com.codahale.metrics.MetricRegistry;
import com.mongodb.MongoClientURI;
import com.mongodb.client.MongoDatabase;
import static java.util.Objects.requireNonNull;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORTED_FILE_PATH;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.INDEXING_PHASE_LOGGER;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Stopwatch;
import org.apache.jackrabbit.guava.common.io.Closer;
import org.apache.jackrabbit.oak.api.CommitFailedException;
Expand All @@ -44,10 +68,10 @@
import org.apache.jackrabbit.oak.plugins.index.FormattingUtils;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
import org.apache.jackrabbit.oak.plugins.index.IndexingReporter;
import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter;
import org.apache.jackrabbit.oak.plugins.index.MetricsUtils;
import org.apache.jackrabbit.oak.plugins.index.NodeTraversalCallback;
import org.apache.jackrabbit.oak.plugins.index.IndexingReporter;
import org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
import org.apache.jackrabbit.oak.plugins.index.progress.MetricRateEstimator;
import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
Expand All @@ -61,37 +85,13 @@
import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static java.util.Objects.requireNonNull;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder.OAK_INDEXER_SORTED_FILE_PATH;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.DEFAULT_OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX;
import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.INDEXING_PHASE_LOGGER;
import com.codahale.metrics.MetricRegistry;
import com.mongodb.ConnectionString;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;

public abstract class DocumentStoreIndexerBase implements Closeable {
public static final String INDEXER_METRICS_PREFIX = "oak_indexer_";
Expand Down Expand Up @@ -483,8 +483,8 @@ private MongoDocumentStore getMongoDocumentStore() {
return requireNonNull(indexHelper.getService(MongoDocumentStore.class));
}

private MongoClientURI getMongoClientURI() {
return requireNonNull(indexHelper.getService(MongoClientURI.class));
private ConnectionString getMongoClientURI() {
return requireNonNull(indexHelper.getService(ConnectionString.class));
}

private MongoDatabase getMongoDatabase() {
Expand All @@ -508,7 +508,8 @@ private void configureEstimators(IndexingProgressReporter progressReporter) {
private long getEstimatedDocumentCount() {
MongoConnection mongoConnection = indexHelper.getService(MongoConnection.class);
if (mongoConnection != null) {
return mongoConnection.getDatabase().getCollection("nodes").count();
MongoCollection<Document> nodesCollection = mongoConnection.getDatabase().getCollection("nodes");
return nodesCollection.countDocuments();
}
return 0;
}
Expand Down
38 changes: 18 additions & 20 deletions ...a/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,20 @@

package org.apache.jackrabbit.oak.index.indexer.document.flatfile;

import com.mongodb.MongoClientURI;
import com.mongodb.client.MongoDatabase;
import static java.util.Collections.unmodifiableSet;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.guava.common.collect.Iterables;
import org.apache.jackrabbit.oak.commons.Compression;
Expand Down Expand Up @@ -51,22 +63,8 @@
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static java.util.Collections.unmodifiableSet;
import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4;
import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP;
import com.mongodb.ConnectionString;
import com.mongodb.client.MongoDatabase;

/**
* This class is where the strategy being selected for building FlatFileStore.
Expand Down Expand Up @@ -112,7 +110,7 @@ public class FlatFileNodeStoreBuilder {
private String checkpoint;
private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP;
private IndexingReporter indexingReporter = IndexingReporter.NOOP;
private MongoClientURI mongoClientURI;
private ConnectionString mongoClientURI;
private boolean withAheadOfTimeBlobDownloading = false;

public enum SortStrategyType {
Expand Down Expand Up @@ -191,7 +189,7 @@ public FlatFileNodeStoreBuilder withCheckpoint(String checkpoint) {
return this;
}

public FlatFileNodeStoreBuilder withMongoClientURI(MongoClientURI mongoClientURI) {
public FlatFileNodeStoreBuilder withMongoClientURI(ConnectionString mongoClientURI) {
this.mongoClientURI = mongoClientURI;
return this;
}
Expand Down
Loading