chore: prepare 2.43.1 (#1754)

SwissDataScienceCenter · Oct 16, 2023 · f6d3b42 · f6d3b42
2 parents 22e8b30 + 0591dd3
commit f6d3b42
Show file tree

Hide file tree

Showing 20 changed files with 136 additions and 94 deletions.
diff --git a/.github/workflows/acceptance-tests.yaml b/.github/workflows/acceptance-tests.yaml
@@ -29,7 +29,7 @@ jobs:
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.8.3
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.8.4
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -42,7 +42,7 @@ jobs:
       name: renku-ci-gr-${{ github.event.number }}
     steps:
       - name: deploy-pr
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.8.3
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.8.4
         env:
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
           DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
@@ -86,7 +86,7 @@ jobs:
     if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
     needs: [check-deploy, deploy-pr]
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.8.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.8.4
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-ci-gr-${{ github.event.number }}
@@ -115,7 +115,7 @@ jobs:
     steps:
       - name: Extract Renku repository reference
         run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
-      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.8.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.8.4
         with:
           e2e-target: ${{ matrix.tests }}
           renku-reference: ${{ env.RENKU_REFERENCE }}
@@ -128,7 +128,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: renku teardown
-        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.8.3
+        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.8.4
         env:
           HELM_RELEASE_REGEX: "^renku-ci-gr-${{ github.event.number }}$"
           GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml
@@ -101,7 +101,7 @@ jobs:
       name: renku-kg-dev
     steps:
       - name: deploy-development
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.8.3
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.8.4
         env:
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
           DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
@@ -125,7 +125,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: deploy-to-graph
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.8.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.8.4
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-kg-dev

diff --git a/.github/workflows/test-and-publish.yml b/.github/workflows/test-and-publish.yml
@@ -107,12 +107,12 @@ jobs:
           echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
           echo "GIT_EMAIL=renku@datascience.ch" >> $GITHUB_ENV
       - name: Build and push images and chart
-        uses: SwissDataScienceCenter/renku-actions/publish-chartpress-images@v1.8.3
+        uses: SwissDataScienceCenter/renku-actions/publish-chartpress-images@v1.8.4
         env:
           DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
       - name: Update component version
-        uses: SwissDataScienceCenter/renku-actions/update-component-version@v1.8.3
+        uses: SwissDataScienceCenter/renku-actions/update-component-version@v1.8.4
         env:
           COMPONENT_NAME: renku-graph
           GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
diff --git a/graph-commons/src/main/resources/projects-ds.ttl b/graph-commons/src/main/resources/projects-ds.ttl
@@ -29,10 +29,6 @@
 <#indexLucene> a text:TextIndexLucene ;
   text:directory <file:///fuseki/databases/projects/lucene_index> ;
   text:entityMap <#entMap> ;
-  text:analyzer [
-    a text:ConfigurableAnalyzer ;
-    text:tokenizer text:LetterTokenizer
-  ];
 .
 
 <#entMap> a         text:EntityMap ;

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -17,7 +17,7 @@ object Dependencies {
     val fs2                    = "3.9.2"
     val http4s                 = "0.23.23"
     val http4sEmber            = "0.23.23"
-    val http4sPrometheus       = "0.24.4"
+    val http4sPrometheus       = "0.24.5"
     val ip4s                   = "3.3.0"
     val jsonld4s               = "0.12.0"
     val log4cats               = "2.6.0"
@@ -26,15 +26,15 @@ object Dependencies {
     val luceneQueryParser      = "9.8.0"
     val owlapi                 = "5.5.0"
     val pureconfig             = "0.17.4"
-    val rdf4jQueryParserSparql = "4.3.6"
+    val rdf4jQueryParserSparql = "4.3.7"
     val refined                = "0.11.0"
     val refinedPureconfig      = "0.11.0"
     val scalacheck             = "1.17.0"
     val scalamock              = "5.2.0"
     val scalatest              = "3.2.17"
     val scalatestScalacheck    = "3.2.14.0"
-    val sentryLogback          = "6.30.0"
-    val skunk                  = "0.6.0"
+    val sentryLogback          = "6.31.0"
+    val skunk                  = "0.6.1"
     val swaggerParser          = "2.1.16"
     val testContainersScala    = "0.41.0"
     val widoco                 = "1.4.20"

diff --git a/triples-generator/Dockerfile b/triples-generator/Dockerfile
@@ -45,7 +45,7 @@ USER tguser
 ENV PATH=$PATH:/home/tguser/.local/bin
 
 # Installing Renku
-RUN python3 -m pip install 'renku==2.7.0rc3' 'sentry-sdk==1.5.11'
+RUN python3 -m pip install 'renku==2.7.0' 'sentry-sdk==1.5.11'
 
 RUN git config --global user.name 'renku'  && \
     git config --global user.email 'renku@renkulab.io'  && \

diff --git a/triples-generator/src/main/resources/application.conf b/triples-generator/src/main/resources/application.conf
@@ -19,7 +19,7 @@ triples-generation = "renku-log"
 # Defines expected version of the renku cli and the schema that is generated by the CLI.
 compatibility {
   # The expected version of CLI used by TS.
-  cli-version = "2.7.0rc3"
+  cli-version = "2.7.0"
 
   # The expected version of the schema as returned by CLI.
   schema-version = "10"

diff --git a/...io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala b/...io/renku/triplesgenerator/events/consumers/tsmigrationrequest/migrations/Migrations.scala
@@ -50,7 +50,7 @@ private[tsmigrationrequest] object Migrations {
     addProjectSlug                 <- projectslug.AddProjectSlug[F]
     datasetsGraphPersonRemover     <- DatasetsGraphPersonRemover[F]
     projectsGraphPersonRemover     <- ProjectsGraphPersonRemover[F]
-    reindexLucene                  <- lucenereindex.ReindexLucene[F]
+    reindexLuceneStdTokenizer      <- lucenereindex.ReindexLucene[F](suffix = "- std tokenizer")
     migrations <- validateNames(
                     datasetsCreator,
                     datasetsRemover,
@@ -69,7 +69,7 @@ private[tsmigrationrequest] object Migrations {
                     addProjectSlug,
                     datasetsGraphPersonRemover,
                     projectsGraphPersonRemover,
-                    reindexLucene
+                    reindexLuceneStdTokenizer
                   )
   } yield migrations
 

diff --git a/...nerator/events/consumers/tsmigrationrequest/migrations/lucenereindex/BacklogCreator.scala b/...nerator/events/consumers/tsmigrationrequest/migrations/lucenereindex/BacklogCreator.scala
@@ -25,6 +25,7 @@ import io.renku.graph.config.RenkuUrlLoader
 import io.renku.graph.model.Schemas._
 import io.renku.graph.model.{RenkuUrl, projects}
 import io.renku.jsonld.syntax._
+import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.Migration
 import io.renku.triplesstore._
 import io.renku.triplesstore.client.model.Triple
 import io.renku.triplesstore.client.syntax._
@@ -36,27 +37,31 @@ private trait BacklogCreator[F[_]] {
 }
 
 private[lucenereindex] object BacklogCreator {
-  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[BacklogCreator[F]] = for {
+
+  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder](migrationName: Migration.Name): F[BacklogCreator[F]] = for {
     implicit0(ru: RenkuUrl) <- RenkuUrlLoader[F]()
     allProjects             <- ProjectsConnectionConfig[F]().map(AllProjects[F](_))
     migrationsDSClient      <- MigrationsConnectionConfig[F]().map(TSClient[F](_))
-  } yield new BacklogCreatorImpl[F](allProjects, migrationsDSClient)
+  } yield new BacklogCreatorImpl[F](migrationName, allProjects, migrationsDSClient)
 
-  def asToBeMigratedInserts(slug: projects.Slug)(implicit ru: RenkuUrl): SparqlQuery =
-    toInsertQuery(toTriples(slug))
+  def asToBeMigratedInserts(migrationName: Migration.Name, slug: projects.Slug)(implicit ru: RenkuUrl): SparqlQuery =
+    toInsertQuery(migrationName, toTriples(migrationName, slug))
 
-  private def toTriples(slug: projects.Slug)(implicit ru: RenkuUrl): Triple =
-    Triple(ReindexLucene.name.asEntityId, renku / "toBeMigrated", slug.asObject)
+  private def toTriples(migrationName: Migration.Name, slug: projects.Slug)(implicit ru: RenkuUrl): Triple =
+    Triple(migrationName.asEntityId, renku / "toBeMigrated", slug.asObject)
 
-  private def toInsertQuery(triple: Triple): SparqlQuery =
+  private def toInsertQuery(migrationName: Migration.Name, triple: Triple): SparqlQuery =
     SparqlQuery
       .ofUnsafe(
-        show"${ReindexLucene.name} - store to backlog",
+        show"$migrationName - store to backlog",
         sparql"INSERT DATA {$triple}"
       )
 }
 
-private class BacklogCreatorImpl[F[_]: Async](allProjects: AllProjects[F], migrationsDSClient: TSClient[F])(implicit
+private class BacklogCreatorImpl[F[_]: Async](migrationName: Migration.Name,
+                                              allProjects:        AllProjects[F],
+                                              migrationsDSClient: TSClient[F]
+)(implicit
     ru: RenkuUrl
 ) extends BacklogCreator[F] {
 
@@ -67,7 +72,7 @@ private class BacklogCreatorImpl[F[_]: Async](allProjects: AllProjects[F], migra
   override def createBacklog(): F[Unit] =
     allProjects
       .findAll(pageSize)
-      .map(md => asToBeMigratedInserts(md.slug))
+      .map(md => asToBeMigratedInserts(migrationName, md.slug))
       .evalMap(migrationsDSClient.updateWithNoResult)
       .compile
       .drain

diff --git a/...ents/consumers/tsmigrationrequest/migrations/lucenereindex/MigrationStartTimeFinder.scala b/...ents/consumers/tsmigrationrequest/migrations/lucenereindex/MigrationStartTimeFinder.scala
@@ -26,8 +26,9 @@ import io.renku.graph.config.RenkuUrlLoader
 import io.renku.graph.model.RenkuUrl
 import io.renku.graph.model.Schemas.renku
 import io.renku.jsonld.syntax._
-import io.renku.triplesstore._
+import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.Migration
 import io.renku.triplesstore.SparqlQuery.Prefixes
+import io.renku.triplesstore._
 import io.renku.triplesstore.client.model.Triple
 import io.renku.triplesstore.client.syntax._
 import org.typelevel.log4cats.Logger
@@ -39,14 +40,16 @@ private trait MigrationStartTimeFinder[F[_]] {
 }
 
 private object MigrationStartTimeFinder {
-  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[MigrationStartTimeFinder[F]] = for {
+  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder](
+      migrationName: Migration.Name
+  ): F[MigrationStartTimeFinder[F]] = for {
     implicit0(ru: RenkuUrl) <- RenkuUrlLoader[F]()
     tsClient                <- MigrationsConnectionConfig[F]().map(TSClient[F](_))
-  } yield new MigrationStartTimeFinderImpl[F](tsClient)
+  } yield new MigrationStartTimeFinderImpl[F](migrationName, tsClient)
 }
 
-private class MigrationStartTimeFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder](
-    tsClient: TSClient[F]
+private class MigrationStartTimeFinderImpl[F[_]: Async: Logger: SparqlQueryTimeRecorder](migrationName: Migration.Name,
+                                                                                         tsClient: TSClient[F]
 )(implicit ru: RenkuUrl)
     extends MigrationStartTimeFinder[F] {
 
@@ -62,23 +65,23 @@ private class MigrationStartTimeFinderImpl[F[_]: Async: Logger: SparqlQueryTimeR
 
   private lazy val startTimeQuery =
     SparqlQuery.ofUnsafe(
-      show"${ReindexLucene.name} - find start time",
+      show"$migrationName - find start time",
       Prefixes of renku -> "renku",
-      s"""|SELECT ?time
-          |WHERE {
-          |  ${ReindexLucene.name.asEntityId.asSparql.sparql} renku:startTime ?time
-          |}
-          |""".stripMargin
+      sparql"""|SELECT ?time
+               |WHERE {
+               |  ${migrationName.asEntityId} renku:startTime ?time
+               |}
+               |""".stripMargin
     )
 
   private implicit lazy val timeDecoder: Decoder[Option[Instant]] = ResultsDecoder[Option, Instant] { implicit cur =>
     extract[Instant]("time")
   }
 
   private def startTimeTriple(instant: Instant) = {
-    val triple = Triple(ReindexLucene.name.asEntityId, renku / "startTime", instant.asTripleObject)
+    val triple = Triple(migrationName.asEntityId, renku / "startTime", instant.asTripleObject)
     SparqlQuery.ofUnsafe(
-      show"${ReindexLucene.name} - store start time",
+      show"$migrationName - store start time",
       s"INSERT DATA {${triple.asSparql.sparql}}"
     )
   }

diff --git a/...nerator/events/consumers/tsmigrationrequest/migrations/lucenereindex/ProgressFinder.scala b/...nerator/events/consumers/tsmigrationrequest/migrations/lucenereindex/ProgressFinder.scala
@@ -26,6 +26,7 @@ import io.circe.Decoder
 import io.renku.graph.config.RenkuUrlLoader
 import io.renku.graph.model.RenkuUrl
 import io.renku.graph.model.Schemas.renku
+import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.Migration
 import io.renku.triplesstore.SparqlQuery.Prefixes
 import io.renku.triplesstore._
 import org.typelevel.log4cats.Logger
@@ -36,14 +37,15 @@ private trait ProgressFinder[F[_]] {
 }
 
 private object ProgressFinder {
-  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[ProgressFinder[F]] = for {
+  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder](migrationName: Migration.Name): F[ProgressFinder[F]] = for {
     implicit0(ru: RenkuUrl) <- RenkuUrlLoader[F]()
     migrationsDSClient      <- MigrationsConnectionConfig[F]().map(TSClient[F](_))
-  } yield new ProgressFinderImpl[F](migrationsDSClient)
+  } yield new ProgressFinderImpl[F](migrationName, migrationsDSClient)
 }
 
-private class ProgressFinderImpl[F[_]: Sync](migrationsDSClient: TSClient[F])(implicit ru: RenkuUrl)
-    extends ProgressFinder[F] {
+private class ProgressFinderImpl[F[_]: Sync](migrationName: Migration.Name, migrationsDSClient: TSClient[F])(implicit
+    ru: RenkuUrl
+) extends ProgressFinder[F] {
 
   import io.renku.jsonld.syntax._
   import io.renku.triplesstore.ResultsDecoder._
@@ -59,11 +61,11 @@ private class ProgressFinderImpl[F[_]: Sync](migrationsDSClient: TSClient[F])(im
   override def findLeftInBacklog: F[Int] =
     migrationsDSClient.queryExpecting[Int](
       SparqlQuery.ofUnsafe(
-        show"${ReindexLucene.name} - find left",
+        show"$migrationName - find left",
         Prefixes of (renku -> "renku"),
         sparql"""|SELECT (COUNT(?slug) AS ?count)
                  |WHERE {
-                 |  ${ReindexLucene.name.asEntityId} renku:toBeMigrated ?slug
+                 |  ${migrationName.asEntityId} renku:toBeMigrated ?slug
                  |}
                  |""".stripMargin
       )

diff --git a/...r/events/consumers/tsmigrationrequest/migrations/lucenereindex/ProjectDonePersister.scala b/...r/events/consumers/tsmigrationrequest/migrations/lucenereindex/ProjectDonePersister.scala
@@ -22,6 +22,7 @@ import cats.effect.Async
 import cats.syntax.all._
 import io.renku.graph.config.RenkuUrlLoader
 import io.renku.graph.model.{RenkuUrl, projects}
+import io.renku.triplesgenerator.events.consumers.tsmigrationrequest.Migration
 import io.renku.triplesstore.{MigrationsConnectionConfig, SparqlQueryTimeRecorder, TSClient}
 import org.typelevel.log4cats.Logger
 
@@ -30,14 +31,16 @@ private trait ProjectDonePersister[F[_]] {
 }
 
 private object ProjectDonePersister {
-  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder]: F[ProjectDonePersister[F]] = for {
-    implicit0(ru: RenkuUrl) <- RenkuUrlLoader[F]()
-    tsClient                <- MigrationsConnectionConfig[F]().map(TSClient[F](_))
-  } yield new ProjectDonePersisterImpl[F](tsClient)
+  def apply[F[_]: Async: Logger: SparqlQueryTimeRecorder](migrationName: Migration.Name): F[ProjectDonePersister[F]] =
+    for {
+      implicit0(ru: RenkuUrl) <- RenkuUrlLoader[F]()
+      tsClient                <- MigrationsConnectionConfig[F]().map(TSClient[F](_))
+    } yield new ProjectDonePersisterImpl[F](migrationName, tsClient)
 }
 
-private class ProjectDonePersisterImpl[F[_]](tsClient: TSClient[F])(implicit ru: RenkuUrl)
-    extends ProjectDonePersister[F] {
+private class ProjectDonePersisterImpl[F[_]](migrationName: Migration.Name, tsClient: TSClient[F])(implicit
+    ru: RenkuUrl
+) extends ProjectDonePersister[F] {
 
   import io.renku.graph.model.Schemas.renku
   import io.renku.jsonld.syntax._
@@ -49,9 +52,9 @@ private class ProjectDonePersisterImpl[F[_]](tsClient: TSClient[F])(implicit ru:
     tsClient.updateWithNoResult(deleteTripleQuery(slug))
 
   private def deleteTripleQuery(slug: projects.Slug) = {
-    val triple = Triple(ReindexLucene.name.asEntityId, renku / "toBeMigrated", slug.asObject)
+    val triple = Triple(migrationName.asEntityId, renku / "toBeMigrated", slug.asObject)
     SparqlQuery.ofUnsafe(
-      show"${ReindexLucene.name} - store migrated",
+      show"$migrationName - store migrated",
       sparql"DELETE DATA {$triple}"
     )
   }