More database nits

eclair-core/src/main/resources/reference.conf

@@ -228,11 +228,12 @@ eclair {
         idle-timeout = 10 minutes
         max-life-time = 30 minutes
       }
+      lock-type = "lease" // lease or none (do not use none in production)
       lease {
         interval = 5 minutes // lease-interval must be greater than lease-renew-interval
         renew-interval = 1 minute
+        lock-timeout = 5 seconds // timeout for the lock statement on the lease table
       }
-      lock-type = "lease" // lease or none
     }
   }
 }

eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala

@@ -211,7 +211,8 @@ object Databases extends Logging {
         val leaseInterval = dbConfig.getDuration("postgres.lease.interval").toSeconds.seconds
         val leaseRenewInterval = dbConfig.getDuration("postgres.lease.renew-interval").toSeconds.seconds
         require(leaseInterval > leaseRenewInterval, "invalid configuration: `db.postgres.lease.interval` must be greater than `db.postgres.lease.renew-interval`")
-        PgLock.LeaseLock(instanceId, leaseInterval, leaseRenewInterval, lockExceptionHandler)
+        val lockTimeout = dbConfig.getDuration("postgres.lease.lock-timeout").toSeconds.seconds
+        PgLock.LeaseLock(instanceId, leaseInterval, leaseRenewInterval, lockTimeout, lockExceptionHandler)
       case unknownLock => throw new RuntimeException(s"unknown postgres lock type: `$unknownLock`")
     }
 

eclair-core/src/main/scala/fr/acinq/eclair/db/pg/PgUtils.scala

@@ -16,6 +16,8 @@
 
 package fr.acinq.eclair.db.pg
 
+import fr.acinq.eclair.db.Monitoring.Metrics._
+import fr.acinq.eclair.db.Monitoring.Tags
 import fr.acinq.eclair.db.jdbc.JdbcUtils
 import fr.acinq.eclair.db.pg.PgUtils.PgLock.LockFailureHandler.LockException
 import grizzled.slf4j.Logging
@@ -104,12 +106,12 @@ object PgUtils extends JdbcUtils {
      *
      * `lockExceptionHandler` provides a lock exception handler to customize the behavior when locking errors occur.
      */
-    case class LeaseLock(instanceId: UUID, leaseDuration: FiniteDuration, leaseRenewInterval: FiniteDuration, lockFailureHandler: LockFailureHandler) extends PgLock {
+    case class LeaseLock(instanceId: UUID, leaseDuration: FiniteDuration, leaseRenewInterval: FiniteDuration, lockTimeout: FiniteDuration, lockFailureHandler: LockFailureHandler) extends PgLock {
 
       import LeaseLock._
 
       override def obtainExclusiveLock(implicit ds: DataSource): Unit = {
-        obtainDatabaseLease(instanceId, leaseDuration) match {
+        obtainDatabaseLease(instanceId, leaseDuration, lockTimeout) match {
           case Right(_) => ()
           case Left(ex) => lockFailureHandler(ex)
         }
@@ -138,15 +140,15 @@ object PgUtils extends JdbcUtils {
       /** We use a [[LeaseLock]] mechanism to get a [[LockLease]]. */
       case class LockLease(expiresAt: Timestamp, instanceId: UUID, expired: Boolean)
 
-      private def obtainDatabaseLease(instanceId: UUID, leaseDuration: FiniteDuration, attempt: Int = 1)(implicit ds: DataSource): Either[LockFailure, LockLease] = synchronized {
+      private def obtainDatabaseLease(instanceId: UUID, leaseDuration: FiniteDuration, lockTimeout: FiniteDuration, attempt: Int = 1)(implicit ds: DataSource): Either[LockFailure, LockLease] = synchronized {
         logger.debug(s"trying to acquire database lease (attempt #$attempt) instance ID=$instanceId")
 
         // this is a recursive method, we need to make sure we don't enter an infinite loop
         if (attempt > 3) return Left(LockFailure.TooManyLockAttempts)
 
         try {
           inTransaction { implicit connection =>
-            acquireExclusiveTableLock()
+            acquireExclusiveTableLock(lockTimeout)
             logger.debug("database lease was successfully acquired")
             checkDatabaseLease(connection, instanceId) match {
               case Right(_) =>
@@ -167,7 +169,7 @@ object PgUtils extends JdbcUtils {
               connection =>
                 logger.warn(s"table $LeaseTable does not exist, trying to create it")
                 initializeLeaseTable(connection)
-                obtainDatabaseLease(instanceId, leaseDuration, attempt + 1)
+                obtainDatabaseLease(instanceId, leaseDuration, lockTimeout, attempt + 1)
             }
           case t: Throwable => Left(LockFailure.GeneralLockException(t))
         }
@@ -181,10 +183,15 @@ object PgUtils extends JdbcUtils {
         }
       }
 
-      private def acquireExclusiveTableLock()(implicit connection: Connection): Unit = {
+      private def acquireExclusiveTableLock(lockTimeout: FiniteDuration)(implicit connection: Connection): Unit = {
         using(connection.createStatement()) {
           statement =>
-            statement.executeUpdate(s"LOCK TABLE $LeaseTable IN ACCESS EXCLUSIVE MODE NOWAIT")
+            // We use a timeout here, because we might not be able to get the lock right away due to concurrent access
+            // by other threads. That timeout gives time for other transactions to complete, then ours can take the lock
+            statement.executeUpdate(s"SET lock_timeout TO '${lockTimeout.toSeconds}s'")
+            withMetrics("utils/lock", Tags.DbBackends.Postgres) {
+              statement.executeUpdate(s"LOCK TABLE $LeaseTable IN ACCESS EXCLUSIVE MODE")
+            }
         }
       }
 

eclair-core/src/test/scala/fr/acinq/eclair/TestDatabases.scala

@@ -59,7 +59,7 @@ object TestDatabases {
     val datasource: DataSource = pg.getPostgresDatabase
     val hikariConfig = new HikariConfig
     hikariConfig.setDataSource(datasource)
-    val lock: PgLock.LeaseLock = PgLock.LeaseLock(UUID.randomUUID(), 10 minutes, 8 minute, LockFailureHandler.logAndThrow)
+    val lock: PgLock.LeaseLock = PgLock.LeaseLock(UUID.randomUUID(), 10 minutes, 8 minute, 5 seconds, LockFailureHandler.logAndThrow)
 
     val jdbcUrlFile: File = new File(sys.props("tmp.dir"), s"jdbcUrlFile_${UUID.randomUUID()}.tmp")
     jdbcUrlFile.deleteOnExit()

eclair-core/src/test/scala/fr/acinq/eclair/db/PgUtilsSpec.scala

@@ -103,11 +103,12 @@ object PgUtilsSpec extends Logging {
        |    idle-timeout = 10 minutes
        |    max-life-time = 30 minutes
        |  }
+       |  lock-type = "lease" // lease or none (do not use none in production)
        |  lease {
        |    interval = 5 seconds // lease-interval must be greater than lease-renew-interval
        |    renew-interval = 2 seconds
+       |    lock-timeout = 5 seconds // timeout for the lock statement on the lease table
        |  }
-       |  lock-type = "lease" // lease or none
        |}
        |""".stripMargin
   )