Add DATA_CLOSED class when active channel is closed (#3170)

We introduce a `DATA_CLOSED` class with minimal information about a
past channel that has been fully closed. This will let us deprecate
legacy channels without having backwards-compatibility issues with
very old closed channels inside our DB.

When channels have never been properly opened or used, we don't
bother storing them in our DB, as it would open the door to DoS
attacks.

We create a dedicated table to store `DATA_CLOSED`. We migrate the
existing DB and remove the foreign key constraint on `htlc_infos`.

Author: t-bast

docs/release-notes/eclair-vnext.md

@@ -11,6 +11,15 @@
 We remove the code used to deserialize channel data from versions of eclair prior to v0.13.
 Node operators running a version of `eclair` older than v0.13 must first upgrade to v0.13 to migrate their channel data, and then upgrade to the latest version.
 
+### Move closed channels to dedicated database table
+
+We previously kept closed channels in the same database table as active channels, with a flag indicating that it was closed.
+This creates performance issues for nodes with a large history of channels, and creates backwards-compatibility issues when changing the channel data format.
+
+We now store closed channels in a dedicated table, where we only keep relevant information regarding the channel.
+When restarting your node, the channels table will automatically be cleaned up and closed channels will move to the new table.
+This may take some time depending on your channels history, but will only happen once.
+
 ### Update minimal version of Bitcoin Core
 
 With this release, eclair requires using Bitcoin Core 29.1.
@@ -22,7 +31,7 @@ Newer versions of Bitcoin Core may be used, but have not been extensively tested
 
 ### API changes
 
-<insert changes>
+- the `closedchannels` API now returns human-readable channel data
 
 ### Miscellaneous improvements and bug fixes
 

eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala

@@ -47,7 +47,6 @@ import fr.acinq.eclair.payment.send.PaymentInitiator._
 import fr.acinq.eclair.payment.send.{ClearRecipient, OfferPayment, PaymentIdentifier}
 import fr.acinq.eclair.router.Router
 import fr.acinq.eclair.router.Router._
-import fr.acinq.eclair.transactions.Transactions.CommitmentFormat
 import fr.acinq.eclair.wire.protocol.OfferTypes.Offer
 import fr.acinq.eclair.wire.protocol._
 import grizzled.slf4j.Logging
@@ -117,7 +116,7 @@ trait Eclair {
 
   def channelInfo(channel: ApiTypes.ChannelIdentifier)(implicit timeout: Timeout): Future[CommandResponse[CMD_GET_CHANNEL_INFO]]
 
-  def closedChannels(nodeId_opt: Option[PublicKey], paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Iterable[RES_GET_CHANNEL_INFO]]
+  def closedChannels(nodeId_opt: Option[PublicKey], paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Iterable[DATA_CLOSED]]
 
   def peers()(implicit timeout: Timeout): Future[Iterable[PeerInfo]]
 
@@ -348,11 +347,9 @@ class EclairImpl(val appKit: Kit) extends Eclair with Logging with SpendFromChan
     sendToChannelTyped(channel = channel, cmdBuilder = CMD_GET_CHANNEL_INFO(_))
   }
 
-  override def closedChannels(nodeId_opt: Option[PublicKey], paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Iterable[RES_GET_CHANNEL_INFO]] = {
+  override def closedChannels(nodeId_opt: Option[PublicKey], paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Iterable[DATA_CLOSED]] = {
     Future {
-      appKit.nodeParams.db.channels.listClosedChannels(nodeId_opt, paginated_opt).map { data =>
-        RES_GET_CHANNEL_INFO(nodeId = data.remoteNodeId, channelId = data.channelId, channel = ActorRef.noSender, state = CLOSED, data = data)
-      }
+      appKit.nodeParams.db.channels.listClosedChannels(nodeId_opt, paginated_opt)
     }
   }
 

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelData.scala

@@ -20,6 +20,7 @@ import akka.actor.{ActorRef, PossiblyHarmful, typed}
 import fr.acinq.bitcoin.scalacompat.Crypto.PublicKey
 import fr.acinq.bitcoin.scalacompat.{ByteVector32, DeterministicWallet, OutPoint, Satoshi, SatoshiLong, Transaction, TxId, TxOut}
 import fr.acinq.eclair.blockchain.fee.{ConfirmationTarget, FeeratePerKw}
+import fr.acinq.eclair.channel.Helpers.Closing
 import fr.acinq.eclair.channel.fund.InteractiveTxBuilder._
 import fr.acinq.eclair.channel.fund.{InteractiveTxBuilder, InteractiveTxSigningSession}
 import fr.acinq.eclair.io.Peer
@@ -560,6 +561,8 @@ sealed trait ChannelDataWithCommitments extends PersistentChannelData {
   def commitments: Commitments
 }
 
+sealed trait ClosedData extends ChannelData
+
 final case class DATA_WAIT_FOR_OPEN_CHANNEL(initFundee: INPUT_INIT_CHANNEL_NON_INITIATOR) extends TransientChannelData {
   val channelId: ByteVector32 = initFundee.temporaryChannelId
 }
@@ -696,6 +699,102 @@ final case class DATA_CLOSING(commitments: Commitments,
 
 final case class DATA_WAIT_FOR_REMOTE_PUBLISH_FUTURE_COMMITMENT(commitments: Commitments, remoteChannelReestablish: ChannelReestablish) extends ChannelDataWithCommitments
 
+/** We use this class when a channel shouldn't be stored in the DB (e.g. because it never confirmed). */
+case class IgnoreClosedData(previousData: ChannelData) extends ClosedData {
+  val channelId: ByteVector32 = previousData.channelId
+}
+
+/**
+ * This class contains the data we will keep in our DB for every closed channel.
+ * It shouldn't contain data we may wish to remove in the future, otherwise we'll have backwards-compatibility issues.
+ * This is why for example the commitmentFormat is a string instead of using the [[CommitmentFormat]] trait, to allow
+ * storing legacy cases that we don't support anymore for active channels.
+ *
+ * Note that we only store channels that have been fully opened and for which we had something at stake. Channels that
+ * are cancelled before having a confirmed funding transactions are ignored, which protects against spam.
+ */
+final case class DATA_CLOSED(channelId: ByteVector32,
+                             remoteNodeId: PublicKey,
+                             fundingTxId: TxId,
+                             fundingOutputIndex: Long,
+                             fundingTxIndex: Long,
+                             fundingKeyPath: String,
+                             channelFeatures: String,
+                             isChannelOpener: Boolean,
+                             commitmentFormat: String,
+                             announced: Boolean,
+                             capacity: Satoshi,
+                             closingTxId: TxId,
+                             closingType: String,
+                             closingScript: ByteVector,
+                             localBalance: MilliSatoshi,
+                             remoteBalance: MilliSatoshi,
+                             closingAmount: Satoshi) extends ClosedData
+
+object DATA_CLOSED {
+  def apply(d: DATA_NEGOTIATING_SIMPLE, closingTx: ClosingTx): DATA_CLOSED = DATA_CLOSED(
+    channelId = d.channelId,
+    remoteNodeId = d.remoteNodeId,
+    fundingTxId = d.commitments.latest.fundingTxId,
+    fundingOutputIndex = d.commitments.latest.fundingInput.index,
+    fundingTxIndex = d.commitments.latest.fundingTxIndex,
+    fundingKeyPath = d.commitments.channelParams.localParams.fundingKeyPath.toString(),
+    channelFeatures = d.commitments.channelParams.channelFeatures.toString,
+    isChannelOpener = d.commitments.latest.channelParams.localParams.isChannelOpener,
+    commitmentFormat = d.commitments.latest.commitmentFormat.toString,
+    announced = d.commitments.latest.channelParams.announceChannel,
+    capacity = d.commitments.latest.capacity,
+    closingTxId = closingTx.tx.txid,
+    closingType = Helpers.Closing.MutualClose(closingTx).toString,
+    closingScript = d.localScriptPubKey,
+    localBalance = d.commitments.latest.localCommit.spec.toLocal,
+    remoteBalance = d.commitments.latest.localCommit.spec.toRemote,
+    closingAmount = closingTx.toLocalOutput_opt.map(_.amount).getOrElse(0 sat)
+  )
+
+  def apply(d: DATA_CLOSING, closingType: Helpers.Closing.ClosingType): DATA_CLOSED = DATA_CLOSED(
+    channelId = d.channelId,
+    remoteNodeId = d.remoteNodeId,
+    fundingTxId = d.commitments.latest.fundingTxId,
+    fundingOutputIndex = d.commitments.latest.fundingInput.index,
+    fundingTxIndex = d.commitments.latest.fundingTxIndex,
+    fundingKeyPath = d.commitments.channelParams.localParams.fundingKeyPath.toString(),
+    channelFeatures = d.commitments.channelParams.channelFeatures.toString,
+    isChannelOpener = d.commitments.latest.channelParams.localParams.isChannelOpener,
+    commitmentFormat = d.commitments.latest.commitmentFormat.toString,
+    announced = d.commitments.latest.channelParams.announceChannel,
+    capacity = d.commitments.latest.capacity,
+    closingTxId = closingType match {
+      case Closing.MutualClose(closingTx) => closingTx.tx.txid
+      case Closing.LocalClose(_, localCommitPublished) => localCommitPublished.commitTx.txid
+      case Closing.CurrentRemoteClose(_, remoteCommitPublished) => remoteCommitPublished.commitTx.txid
+      case Closing.NextRemoteClose(_, remoteCommitPublished) => remoteCommitPublished.commitTx.txid
+      case Closing.RecoveryClose(remoteCommitPublished) => remoteCommitPublished.commitTx.txid
+      case Closing.RevokedClose(revokedCommitPublished) => revokedCommitPublished.commitTx.txid
+    },
+    closingType = closingType.toString,
+    closingScript = d.finalScriptPubKey,
+    localBalance = closingType match {
+      case _: Closing.CurrentRemoteClose => d.commitments.latest.remoteCommit.spec.toRemote
+      case _: Closing.NextRemoteClose => d.commitments.latest.nextRemoteCommit_opt.getOrElse(d.commitments.latest.remoteCommit).spec.toRemote
+      case _ => d.commitments.latest.localCommit.spec.toLocal
+    },
+    remoteBalance = closingType match {
+      case _: Closing.CurrentRemoteClose => d.commitments.latest.remoteCommit.spec.toLocal
+      case _: Closing.NextRemoteClose => d.commitments.latest.nextRemoteCommit_opt.getOrElse(d.commitments.latest.remoteCommit).spec.toLocal
+      case _ => d.commitments.latest.localCommit.spec.toRemote
+    },
+    closingAmount = closingType match {
+      case Closing.MutualClose(closingTx) => closingTx.toLocalOutput_opt.map(_.amount).getOrElse(0 sat)
+      case Closing.LocalClose(_, localCommitPublished) => Closing.closingBalance(d.channelParams, d.commitments.latest.commitmentFormat, d.finalScriptPubKey, localCommitPublished)
+      case Closing.CurrentRemoteClose(_, remoteCommitPublished) => Closing.closingBalance(d.channelParams, d.commitments.latest.commitmentFormat, d.finalScriptPubKey, remoteCommitPublished)
+      case Closing.NextRemoteClose(_, remoteCommitPublished) => Closing.closingBalance(d.channelParams, d.commitments.latest.commitmentFormat, d.finalScriptPubKey, remoteCommitPublished)
+      case Closing.RecoveryClose(remoteCommitPublished) => Closing.closingBalance(d.channelParams, d.commitments.latest.commitmentFormat, d.finalScriptPubKey, remoteCommitPublished)
+      case Closing.RevokedClose(revokedCommitPublished) => Closing.closingBalance(d.channelParams, d.commitments.latest.commitmentFormat, d.finalScriptPubKey, revokedCommitPublished)
+    }
+  )
+}
+
 /** Local params that apply for the channel's lifetime. */
 case class LocalChannelParams(nodeId: PublicKey,
                               fundingKeyPath: DeterministicWallet.KeyPath,

eclair-core/src/main/scala/fr/acinq/eclair/channel/Helpers.scala

@@ -608,13 +608,13 @@ object Helpers {
 
     // @formatter:off
     sealed trait ClosingType
-    case class MutualClose(tx: ClosingTx) extends ClosingType
-    case class LocalClose(localCommit: LocalCommit, localCommitPublished: LocalCommitPublished) extends ClosingType
+    case class MutualClose(tx: ClosingTx) extends ClosingType { override def toString: String = "mutual-close" }
+    case class LocalClose(localCommit: LocalCommit, localCommitPublished: LocalCommitPublished) extends ClosingType { override def toString: String = "local-close" }
     sealed trait RemoteClose extends ClosingType { def remoteCommit: RemoteCommit; def remoteCommitPublished: RemoteCommitPublished }
-    case class CurrentRemoteClose(remoteCommit: RemoteCommit, remoteCommitPublished: RemoteCommitPublished) extends RemoteClose
-    case class NextRemoteClose(remoteCommit: RemoteCommit, remoteCommitPublished: RemoteCommitPublished) extends RemoteClose
-    case class RecoveryClose(remoteCommitPublished: RemoteCommitPublished) extends ClosingType
-    case class RevokedClose(revokedCommitPublished: RevokedCommitPublished) extends ClosingType
+    case class CurrentRemoteClose(remoteCommit: RemoteCommit, remoteCommitPublished: RemoteCommitPublished) extends RemoteClose { override def toString: String = "remote-close" }
+    case class NextRemoteClose(remoteCommit: RemoteCommit, remoteCommitPublished: RemoteCommitPublished) extends RemoteClose { override def toString: String = "next-remote-close" }
+    case class RecoveryClose(remoteCommitPublished: RemoteCommitPublished) extends ClosingType { override def toString: String = "recovery-close" }
+    case class RevokedClose(revokedCommitPublished: RevokedCommitPublished) extends ClosingType { override def toString: String = "revoked-close" }
     // @formatter:on
 
     /**
@@ -1707,6 +1707,23 @@ object Helpers {
       revokedCommitPublished.copy(irrevocablySpent = revokedCommitPublished.irrevocablySpent ++ relevantOutpoints.map(o => o -> tx).toMap)
     }
 
+    /** Returns the amount we've successfully claimed from a force-closed channel. */
+    def closingBalance(channelParams: ChannelParams, commitmentFormat: CommitmentFormat, closingScript: ByteVector, commit: CommitPublished): Satoshi = {
+      val toLocal = commit.localOutput_opt match {
+        case Some(o) if o.index < commit.commitTx.txOut.size => commit.commitTx.txOut(o.index.toInt).amount
+        case _ => 0 sat
+      }
+      val toClosingScript = commit.irrevocablySpent.values.flatMap(_.txOut)
+        .filter(_.publicKeyScript == closingScript)
+        .map(_.amount)
+        .sum
+      commitmentFormat match {
+        case DefaultCommitmentFormat if channelParams.localParams.walletStaticPaymentBasepoint.nonEmpty => toLocal + toClosingScript
+        case DefaultCommitmentFormat => toClosingScript
+        case _: AnchorOutputsCommitmentFormat | _: SimpleTaprootChannelCommitmentFormat => toClosingScript
+      }
+    }
+
   }
 
 }

eclair-core/src/main/scala/fr/acinq/eclair/channel/fsm/Channel.scala

@@ -385,7 +385,7 @@ class Channel(val nodeParams: NodeParams, val channelKeys: ChannelKeys, val wall
         case closing: DATA_CLOSING if Closing.nothingAtStake(closing) =>
           log.info("we have nothing at stake, going straight to CLOSED")
           context.system.eventStream.publish(ChannelAborted(self, remoteNodeId, closing.channelId))
-          goto(CLOSED) using closing
+          goto(CLOSED) using IgnoreClosedData(closing)
         case closing: DATA_CLOSING =>
           val localPaysClosingFees = closing.commitments.localChannelParams.paysClosingFees
           val closingType_opt = Closing.isClosingTypeAlreadyKnown(closing)
@@ -2289,7 +2289,7 @@ class Channel(val nodeParams: NodeParams, val channelKeys: ChannelKeys, val wall
         case Some(closingType) =>
           log.info("channel closed (type={})", EventType.Closed(closingType).label)
           context.system.eventStream.publish(ChannelClosed(self, d.channelId, closingType, d.commitments))
-          goto(CLOSED) using d1 storing()
+          goto(CLOSED) using DATA_CLOSED(d1, closingType)
         case None =>
           stay() using d1 storing()
       }
@@ -2366,9 +2366,12 @@ class Channel(val nodeParams: NodeParams, val channelKeys: ChannelKeys, val wall
   when(CLOSED)(handleExceptions {
     case Event(Symbol("shutdown"), _) =>
       stateData match {
-        case d: PersistentChannelData =>
-          log.info(s"deleting database record for channelId=${d.channelId}")
-          nodeParams.db.channels.removeChannel(d.channelId)
+        case d: DATA_CLOSED =>
+          log.info(s"moving channelId=${d.channelId} to the closed channels DB")
+          nodeParams.db.channels.removeChannel(d.channelId, Some(d))
+        case _: PersistentChannelData | _: IgnoreClosedData =>
+          log.info("deleting database record for channelId={}", stateData.channelId)
+          nodeParams.db.channels.removeChannel(stateData.channelId, None)
         case _: TransientChannelData => // nothing was stored in the DB
       }
       log.info("shutting down")
@@ -3029,10 +3032,11 @@ class Channel(val nodeParams: NodeParams, val channelKeys: ChannelKeys, val wall
       }
 
     case Event(WatchTxConfirmedTriggered(_, _, tx), d: DATA_NEGOTIATING_SIMPLE) if d.findClosingTx(tx).nonEmpty =>
-      val closingType = MutualClose(d.findClosingTx(tx).get)
+      val closingTx = d.findClosingTx(tx).get
+      val closingType = MutualClose(closingTx)
       log.info("channel closed (type={})", EventType.Closed(closingType).label)
       context.system.eventStream.publish(ChannelClosed(self, d.channelId, closingType, d.commitments))
-      goto(CLOSED) using d storing()
+      goto(CLOSED) using DATA_CLOSED(d, closingTx)
 
     case Event(WatchFundingSpentTriggered(tx), d: ChannelDataWithCommitments) =>
       if (d.commitments.all.map(_.fundingTxId).contains(tx.txid)) {
@@ -3070,6 +3074,7 @@ class Channel(val nodeParams: NodeParams, val channelKeys: ChannelKeys, val wall
           case d: ChannelDataWithCommitments => Some(d.commitments)
           case _: ChannelDataWithoutCommitments => None
           case _: TransientChannelData => None
+          case _: ClosedData => None
         }
         context.system.eventStream.publish(ChannelStateChanged(self, nextStateData.channelId, peer, remoteNodeId, state, nextState, commitments_opt))
       }