Skip to content

Revert import updates backfilling #1169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1070,7 +1070,7 @@ class DecentralizedSynchronizerMigrationIntegrationTest

val backfilledUpdates =
sv1ScanLocalBackend.appState.store.updateHistory
.getAllUpdates(None, PageLimit.tryCreate(1000))
.getUpdates(None, includeImportUpdates = true, PageLimit.tryCreate(1000))
.futureValue
backfilledUpdates.collect {
case TreeUpdateWithMigrationId(tree, migrationId)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ class ScanHistoryBackfillingIntegrationTest

sv2ScanBackend.appState.store.updateHistory
.getBackfillingState()
.futureValue should be(BackfillingState.InProgress(false, false))
.futureValue should be(BackfillingState.InProgress)
sv2ScanBackend.getBackfillingStatus().complete shouldBe false
assertThrowsAndLogsCommandFailures(
readUpdateHistoryFromScan(sv2ScanBackend),
Expand Down Expand Up @@ -518,7 +518,7 @@ class ScanHistoryBackfillingIntegrationTest
private def allUpdatesFromScanBackend(scanBackend: ScanAppBackendReference) = {
// Need to use the store directly, as the HTTP endpoint refuses to return data unless it's completely backfilled
scanBackend.appState.store.updateHistory
.getAllUpdates(None, PageLimit.tryCreate(1000))
.getUpdates(None, includeImportUpdates = true, PageLimit.tryCreate(1000))
.futureValue
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ trait UpdateHistoryTestUtil extends TestCommon {
updateHistoryFromParticipant(ledgerBegin, updateHistory.updateStreamParty, participant)

val recordedUpdates = updateHistory
.getAllUpdates(
.getUpdates(
Some(
(
0L,
Expand All @@ -104,6 +104,7 @@ trait UpdateHistoryTestUtil extends TestCommon {
actualUpdates.head.update.recordTime.addMicros(-1L),
)
),
includeImportUpdates = true,
PageLimit.tryCreate(actualUpdates.size),
)
.futureValue
Expand Down Expand Up @@ -138,8 +139,9 @@ trait UpdateHistoryTestUtil extends TestCommon {
scanClient: ScanAppClientReference,
): Assertion = {
val historyFromStore = scanBackend.appState.store.updateHistory
.getAllUpdates(
.getUpdates(
None,
includeImportUpdates = true,
PageLimit.tryCreate(1000),
)
.futureValue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,20 +285,15 @@ object HistoryBackfilling {
* None if the given migration id is the beginning of known history.
* @param recordTimeRange All domains that produced history items in the given migration id,
* along with the record time of the newest and oldest history item associated with each domain.
* @param lastImportUpdateId The id of the last import update (where import updates are sorted by update id)
* for the given migration id, if any.
* @param complete True if the backfilling for the given migration id is complete,
* i.e., the history knows the first item for each domain in the given migration id.
* We need this to decide when the backfilling is complete, because it might be difficult to
* identify the first item of a migration otherwise.
* @param importUpdatesComplete True if the import updates for the given migration id are complete.
*/
final case class SourceMigrationInfo(
previousMigrationId: Option[Long],
recordTimeRange: Map[SynchronizerId, DomainRecordTimeRange],
lastImportUpdateId: Option[String],
complete: Boolean,
importUpdatesComplete: Boolean,
)

/** Information about the point at which backfilling is currently inserting data.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,39 +102,6 @@ class HistoryMetrics(metricsFactory: LabeledMetricsFactory)(implicit
)(metricsContext)
}

object ImportUpdatesBackfilling {
private val importUpdatesBackfillingPrefix: MetricName = prefix :+ "import-updates-backfilling"

val latestMigrationId: Gauge[Long] =
metricsFactory.gauge(
MetricInfo(
name = importUpdatesBackfillingPrefix :+ "latest-migration-id",
summary = "The migration id of the latest backfilled import update",
Traffic,
),
initial = -1L,
)(metricsContext)

val contractCount: Counter =
metricsFactory.counter(
MetricInfo(
name = importUpdatesBackfillingPrefix :+ "contract-count",
summary = "The number of contracts that have been backfilled",
Traffic,
)
)(metricsContext)

val completed: Gauge[Int] =
metricsFactory.gauge(
MetricInfo(
name = importUpdatesBackfillingPrefix :+ "completed",
summary = "Whether it was completed (1) or not (0)",
Debug,
),
initial = 0,
)(metricsContext)
}

object UpdateHistory {
private val updateHistoryPrefix: MetricName = prefix :+ "updates"

Expand Down

This file was deleted.

Loading
Loading