Add support for option_payment_metadata

docs/release-notes/eclair-vnext.md

@@ -24,6 +24,15 @@ Eclair now supports the feature `option_onion_messages`. If this feature is enab
 It can also send onion messages with the `sendonionmessage` API.
 Messages sent to Eclair can be read with the websocket API.
 
+### Support for `option_payment_metadata`
+
+Eclair now supports the `option_payment_metadata` feature (see https://github.com/lightning/bolts/pull/912).
+This feature will let recipients generate "light" invoices that don't need to be stored locally until they're paid.
+This is particularly useful for payment hubs that generate a lot of invoices (e.g. to be displayed on a website) but expect only a fraction of them to actually be paid.
+
+Eclair includes a small `payment_metadata` field in all invoices it generates.
+This lets node operators verify that payers actually support that feature.
+
 ### API changes
 
 #### Timestamps

eclair-core/src/main/resources/reference.conf

@@ -58,6 +58,7 @@ eclair {
     option_shutdown_anysegwit = optional
     option_onion_messages = disabled
     option_channel_type = optional
+    option_payment_metadata = optional
     trampoline_payment = disabled
     keysend = disabled
   }

eclair-core/src/main/scala/fr/acinq/eclair/Features.scala

@@ -34,6 +34,8 @@ object FeatureSupport {
 
 trait Feature {
 
+  this: FeatureScope =>
+
   def rfcName: String
   def mandatory: Int
   def optional: Int = mandatory + 1
@@ -46,6 +48,15 @@ trait Feature {
   override def toString = rfcName
 
 }
+
+/** Feature scope as defined in Bolt 9. */
+sealed trait FeatureScope
+/** Feature that should be advertised in init messages. */
+trait InitFeature extends FeatureScope
+/** Feature that should be advertised in node announcements. */
+trait NodeFeature extends FeatureScope
+/** Feature that should be advertised in invoices. */
+trait InvoiceFeature extends FeatureScope
 // @formatter:on
 
 case class UnknownFeature(bitIndex: Int)
@@ -71,6 +82,13 @@ case class Features(activated: Map[Feature, FeatureSupport], unknown: Set[Unknow
     unknownFeaturesOk && knownFeaturesOk
   }
 
+  def initFeatures(): Features = Features(activated.collect { case (f: InitFeature, s) => (f: Feature, s) }, unknown)
+
+  def nodeAnnouncementFeatures(): Features = Features(activated.collect { case (f: NodeFeature, s) => (f: Feature, s) }, unknown)
+
+  // NB: we don't include unknown features in invoices, which means plugins cannot inject invoice features.
+  def invoiceFeatures(): Map[Feature with InvoiceFeature, FeatureSupport] = activated.collect { case (f: InvoiceFeature, s) => (f, s) }
+
   def toByteVector: ByteVector = {
     val activatedFeatureBytes = toByteVectorFromIndex(activated.map { case (feature, support) => feature.supportBit(support) }.toSet)
     val unknownFeatureBytes = toByteVectorFromIndex(unknown.map(_.bitIndex))
@@ -137,91 +155,96 @@ object Features {
     }
   }
 
-  case object OptionDataLossProtect extends Feature {
+  case object OptionDataLossProtect extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_data_loss_protect"
     val mandatory = 0
   }
 
-  case object InitialRoutingSync extends Feature {
+  case object InitialRoutingSync extends Feature with InitFeature {
     val rfcName = "initial_routing_sync"
     // reserved but not used as per lightningnetwork/lightning-rfc/pull/178
     val mandatory = 2
   }
 
-  case object OptionUpfrontShutdownScript extends Feature {
+  case object OptionUpfrontShutdownScript extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_upfront_shutdown_script"
     val mandatory = 4
   }
 
-  case object ChannelRangeQueries extends Feature {
+  case object ChannelRangeQueries extends Feature with InitFeature with NodeFeature {
     val rfcName = "gossip_queries"
     val mandatory = 6
   }
 
-  case object VariableLengthOnion extends Feature {
+  case object VariableLengthOnion extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "var_onion_optin"
     val mandatory = 8
   }
 
-  case object ChannelRangeQueriesExtended extends Feature {
+  case object ChannelRangeQueriesExtended extends Feature with InitFeature with NodeFeature {
     val rfcName = "gossip_queries_ex"
     val mandatory = 10
   }
 
-  case object StaticRemoteKey extends Feature {
+  case object StaticRemoteKey extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_static_remotekey"
     val mandatory = 12
   }
 
-  case object PaymentSecret extends Feature {
+  case object PaymentSecret extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "payment_secret"
     val mandatory = 14
   }
 
-  case object BasicMultiPartPayment extends Feature {
+  case object BasicMultiPartPayment extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "basic_mpp"
     val mandatory = 16
   }
 
-  case object Wumbo extends Feature {
+  case object Wumbo extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_support_large_channel"
     val mandatory = 18
   }
 
-  case object AnchorOutputs extends Feature {
+  case object AnchorOutputs extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_anchor_outputs"
     val mandatory = 20
   }
 
-  case object AnchorOutputsZeroFeeHtlcTx extends Feature {
+  case object AnchorOutputsZeroFeeHtlcTx extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_anchors_zero_fee_htlc_tx"
     val mandatory = 22
   }
 
-  case object ShutdownAnySegwit extends Feature {
+  case object ShutdownAnySegwit extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_shutdown_anysegwit"
     val mandatory = 26
   }
 
-  case object OnionMessages extends Feature {
+  case object OnionMessages extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_onion_messages"
     val mandatory = 38
   }
 
-  case object ChannelType extends Feature {
+  case object ChannelType extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_channel_type"
     val mandatory = 44
   }
 
+  case object PaymentMetadata extends Feature with InvoiceFeature {
+    val rfcName = "option_payment_metadata"
+    val mandatory = 48
+  }
+
   // TODO: @t-bast: update feature bits once spec-ed (currently reserved here: https://github.com/lightningnetwork/lightning-rfc/issues/605)
   // We're not advertising these bits yet in our announcements, clients have to assume support.
   // This is why we haven't added them yet to `areSupported`.
-  case object TrampolinePayment extends Feature {
+  case object TrampolinePayment extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "trampoline_payment"
     val mandatory = 50
   }
 
-  case object KeySend extends Feature {
+  case object KeySend extends Feature with NodeFeature {
     val rfcName = "keysend"
     val mandatory = 54
   }
@@ -242,6 +265,7 @@ object Features {
     ShutdownAnySegwit,
     OnionMessages,
     ChannelType,
+    PaymentMetadata,
     TrampolinePayment,
     KeySend
   )

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -109,7 +109,8 @@ case class NodeParams(nodeKeyManager: NodeKeyManager,
 
   def currentBlockHeight: Long = blockCount.get
 
-  def featuresFor(nodeId: PublicKey): Features = overrideFeatures.getOrElse(nodeId, features)
+  /** Returns the features that should be used in our init message with the given peer. */
+  def initFeaturesFor(nodeId: PublicKey): Features = overrideFeatures.getOrElse(nodeId, features).initFeatures()
 }
 
 object NodeParams extends Logging {

eclair-core/src/main/scala/fr/acinq/eclair/io/Switchboard.scala

@@ -90,7 +90,7 @@ class Switchboard(nodeParams: NodeParams, peerFactory: Switchboard.PeerFactory)
     case authenticated: PeerConnection.Authenticated =>
       // if this is an incoming connection, we might not yet have created the peer
       val peer = createOrGetPeer(authenticated.remoteNodeId, offlineChannels = Set.empty)
-      val features = nodeParams.featuresFor(authenticated.remoteNodeId)
+      val features = nodeParams.initFeaturesFor(authenticated.remoteNodeId)
       // if the peer is whitelisted, we sync with them, otherwise we only sync with peers with whom we have at least one channel
       val doSync = nodeParams.syncWhitelist.contains(authenticated.remoteNodeId) || (nodeParams.syncWhitelist.isEmpty && peersWithChannels.contains(authenticated.remoteNodeId))
       authenticated.peerConnection ! PeerConnection.InitializeConnection(peer, nodeParams.chainHash, features, doSync)

eclair-core/src/main/scala/fr/acinq/eclair/json/JsonSerializers.scala

@@ -334,6 +334,7 @@ object PaymentRequestSerializer extends MinimalSerializer({
         FeatureSupportSerializer +
         UnknownFeatureSerializer
     ))
+    val paymentMetadata = p.paymentMetadata.map(m => JField("paymentMetadata", JString(m.toHex))).toSeq
     val routingInfo = JField("routingInfo", Extraction.decompose(p.routingInfo)(
       DefaultFormats +
         ByteVector32Serializer +
@@ -343,12 +344,14 @@ object PaymentRequestSerializer extends MinimalSerializer({
         MilliSatoshiSerializer +
         CltvExpiryDeltaSerializer
     ))
-    val fieldList = List(JField("prefix", JString(p.prefix)),
+    val fieldList = List(
+      JField("prefix", JString(p.prefix)),
       JField("timestamp", JLong(p.timestamp.toLong)),
       JField("nodeId", JString(p.nodeId.toString())),
       JField("serialized", JString(PaymentRequest.write(p))),
       p.description.fold(string => JField("description", JString(string)), hash => JField("descriptionHash", JString(hash.toHex))),
       JField("paymentHash", JString(p.paymentHash.toString()))) ++
+      paymentMetadata ++
       expiry ++
       minFinalCltvExpiry ++
       amount :+

eclair-core/src/main/scala/fr/acinq/eclair/payment/Monitoring.scala

@@ -27,6 +27,7 @@ object Monitoring {
     val PaymentAmount = Kamon.histogram("payment.amount", "Payment amount (satoshi)")
     val PaymentFees = Kamon.histogram("payment.fees", "Payment fees (satoshi)")
     val PaymentParts = Kamon.histogram("payment.parts", "Number of HTLCs per payment (MPP)")
+    val PaymentHtlcReceived = Kamon.counter("payment.received", "Number of valid htlcs received")
     val PaymentFailed = Kamon.counter("payment.failed", "Number of failed payment")
     val PaymentError = Kamon.counter("payment.error", "Non-fatal errors encountered during payment attempts")
     val PaymentAttempt = Kamon.histogram("payment.attempt", "Number of attempts before a payment succeeds")
@@ -71,6 +72,7 @@ object Monitoring {
     val PaymentId = "paymentId"
     val ParentId = "parentPaymentId"
     val PaymentHash = "paymentHash"
+    val PaymentMetadataIncluded = "paymentMetadataIncluded"
 
     val Amount = "amount"
     val TotalAmount = "totalAmount"

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentPacket.scala

@@ -117,7 +117,7 @@ object IncomingPaymentPacket {
     } else {
       // We merge contents from the outer and inner payloads.
       // We must use the inner payload's total amount and payment secret because the payment may be split between multiple trampoline payments (#reckless).
-      Right(FinalPacket(add, PaymentOnion.createMultiPartPayload(outerPayload.amount, innerPayload.totalAmount, outerPayload.expiry, innerPayload.paymentSecret)))
+      Right(FinalPacket(add, PaymentOnion.createMultiPartPayload(outerPayload.amount, innerPayload.totalAmount, outerPayload.expiry, innerPayload.paymentSecret, innerPayload.paymentMetadata)))
     }
   }
 
@@ -213,9 +213,12 @@ object OutgoingPaymentPacket {
    *         - the trampoline onion to include in final payload of a normal onion
    */
   def buildTrampolineToLegacyPacket(invoice: PaymentRequest, hops: Seq[NodeHop], finalPayload: PaymentOnion.FinalPayload): (MilliSatoshi, CltvExpiry, Sphinx.PacketAndSecrets) = {
-    val (firstAmount, firstExpiry, payloads) = hops.drop(1).reverse.foldLeft((finalPayload.amount, finalPayload.expiry, Seq[PaymentOnion.PerHopPayload](finalPayload))) {
+    // NB: the final payload will never reach the recipient, since the next-to-last node in the trampoline route will convert that to a non-trampoline payment.
+    // We use the smallest final payload possible, otherwise we may overflow the trampoline onion size.
+    val dummyFinalPayload = PaymentOnion.createSinglePartPayload(finalPayload.amount, finalPayload.expiry, finalPayload.paymentSecret, None)
+    val (firstAmount, firstExpiry, payloads) = hops.drop(1).reverse.foldLeft((finalPayload.amount, finalPayload.expiry, Seq[PaymentOnion.PerHopPayload](dummyFinalPayload))) {
       case ((amount, expiry, payloads), hop) =>
-        // The next-to-last trampoline hop must include invoice data to indicate the conversion to a legacy payment.
+        // The next-to-last node in the trampoline route must receive invoice data to indicate the conversion to a non-trampoline payment.
         val payload = if (payloads.length == 1) {
           PaymentOnion.createNodeRelayToNonTrampolinePayload(finalPayload.amount, finalPayload.totalAmount, finalPayload.expiry, hop.nextNodeId, invoice)
         } else {

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentRequest.scala

@@ -19,12 +19,11 @@ package fr.acinq.eclair.payment
 import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
 import fr.acinq.bitcoin.{Base58, Base58Check, Bech32, Block, ByteVector32, ByteVector64, Crypto}
 import fr.acinq.eclair.payment.PaymentRequest._
-import fr.acinq.eclair.{CltvExpiryDelta, FeatureSupport, Features, MilliSatoshi, MilliSatoshiLong, NodeParams, ShortChannelId, TimestampSecond, randomBytes32}
+import fr.acinq.eclair.{CltvExpiryDelta, Feature, FeatureSupport, Features, InvoiceFeature, MilliSatoshi, MilliSatoshiLong, NodeParams, ShortChannelId, TimestampSecond, randomBytes32}
 import scodec.bits.{BitVector, ByteOrdering, ByteVector}
 import scodec.codecs.{list, ubyte}
 import scodec.{Codec, Err}
 
-import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
 
 /**
@@ -67,6 +66,11 @@ case class PaymentRequest(prefix: String, amount: Option[MilliSatoshi], timestam
     case PaymentRequest.DescriptionHash(h) => Right(h)
   }.get
 
+  /**
+   * @return metadata about the payment (see option_payment_metadata).
+   */
+  lazy val paymentMetadata: Option[ByteVector] = tags.collectFirst { case m: PaymentRequest.PaymentMetadata => m.data }
+
   /**
    * @return the fallback address if any. It could be a script address, pubkey address, ..
    */
@@ -126,6 +130,11 @@ object PaymentRequest {
     Block.LivenetGenesisBlock.hash -> "lnbc"
   )
 
+  val defaultFeatures: Map[Feature with InvoiceFeature, FeatureSupport] = Map(
+    Features.VariableLengthOnion -> FeatureSupport.Mandatory,
+    Features.PaymentSecret -> FeatureSupport.Mandatory,
+  )
+
   def apply(chainHash: ByteVector32,
             amount: Option[MilliSatoshi],
             paymentHash: ByteVector32,
@@ -137,14 +146,16 @@ object PaymentRequest {
             extraHops: List[List[ExtraHop]] = Nil,
             timestamp: TimestampSecond = TimestampSecond.now(),
             paymentSecret: ByteVector32 = randomBytes32(),
-            features: PaymentRequestFeatures = PaymentRequestFeatures(Features.VariableLengthOnion.mandatory, Features.PaymentSecret.mandatory)): PaymentRequest = {
+            paymentMetadata: Option[ByteVector] = None,
+            features: PaymentRequestFeatures = PaymentRequestFeatures(defaultFeatures)): PaymentRequest = {
     require(features.requirePaymentSecret, "invoices must require a payment secret")
     val prefix = prefixes(chainHash)
     val tags = {
       val defaultTags = List(
         Some(PaymentHash(paymentHash)),
         Some(description.fold(Description, DescriptionHash)),
         Some(PaymentSecret(paymentSecret)),
+        paymentMetadata.map(PaymentMetadata),
         fallbackAddress.map(FallbackAddress(_)),
         expirySeconds.map(Expiry(_)),
         Some(MinFinalCltvExpiry(minFinalCltvExpiryDelta.toInt)),
@@ -193,7 +204,6 @@ object PaymentRequest {
   case class InvalidTag23(data: BitVector) extends InvalidTaggedField
   case class UnknownTag25(data: BitVector) extends UnknownTaggedField
   case class UnknownTag26(data: BitVector) extends UnknownTaggedField
-  case class UnknownTag27(data: BitVector) extends UnknownTaggedField
   case class UnknownTag28(data: BitVector) extends UnknownTaggedField
   case class UnknownTag29(data: BitVector) extends UnknownTaggedField
   case class UnknownTag30(data: BitVector) extends UnknownTaggedField
@@ -229,6 +239,11 @@ object PaymentRequest {
    */
   case class DescriptionHash(hash: ByteVector32) extends TaggedField
 
+  /**
+   * Additional metadata to attach to the payment.
+   */
+  case class PaymentMetadata(data: ByteVector) extends TaggedField
+
   /**
    * Fallback Payment that specifies a fallback payment address to be used if LN payment cannot be processed
    */
@@ -355,8 +370,8 @@ object PaymentRequest {
   }
 
   object PaymentRequestFeatures {
-    def apply(features: Int*): PaymentRequestFeatures = PaymentRequestFeatures(long2bits(features.foldLeft(0L) {
-      case (current, feature) => current + (1L << feature)
+    def apply(features: Map[Feature with InvoiceFeature, FeatureSupport]): PaymentRequestFeatures = PaymentRequestFeatures(long2bits(features.foldLeft(0L) {
+      case (current, (feature, support)) => current + (1L << feature.supportBit(support))
     }))
   }
 
@@ -429,7 +444,7 @@ object PaymentRequest {
       .typecase(24, dataCodec(bits).as[MinFinalCltvExpiry])
       .typecase(25, dataCodec(bits).as[UnknownTag25])
       .typecase(26, dataCodec(bits).as[UnknownTag26])
-      .typecase(27, dataCodec(bits).as[UnknownTag27])
+      .typecase(27, dataCodec(alignedBytesCodec(bytes)).as[PaymentMetadata])
       .typecase(28, dataCodec(bits).as[UnknownTag28])
       .typecase(29, dataCodec(bits).as[UnknownTag29])
       .typecase(30, dataCodec(bits).as[UnknownTag30])