Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
b68cc0e
logservice,config: add low latency resolved ts mode
asddongmen Jul 13, 2026
50d42b9
downstreamadapter: reduce low latency heartbeat interval
asddongmen Jul 13, 2026
4a37106
maintainer: trigger checkpoint calculation on watermark update
asddongmen Jul 13, 2026
fdf6481
maintainer: report low latency watermarks promptly
asddongmen Jul 13, 2026
33f60bb
eventservice: shorten low latency resolved ts flush
asddongmen Jul 13, 2026
8a651dc
logservice: reduce owner resolved ts metric lag
asddongmen Jul 13, 2026
ddde9c8
logservice: simplify low latency resolved ts advancement
asddongmen Jul 13, 2026
81de98f
eventservice: revert low latency resolved ts flush
asddongmen Jul 14, 2026
deaae7f
logservice: report maximum per-node resolved ts lag
asddongmen Jul 15, 2026
2eadfea
eventservice: retry schema-capped resolved ts
asddongmen Jul 15, 2026
4199300
workload: distribute bank3 writes uniformly
asddongmen Jul 15, 2026
0d79d5b
eventservice,schemastore: make schema wakeup event driven
asddongmen Jul 15, 2026
07fe6a0
eventservice,schemastore: remove schema retry scheduling
asddongmen Jul 21, 2026
18e4091
eventservice: serialize dispatcher scan scheduling
asddongmen Jul 21, 2026
4b11635
eventservice: retry schema-blocked scans periodically
asddongmen Jul 21, 2026
eb3112d
eventservice: continue scans after running notifications
asddongmen Jul 22, 2026
104b175
workload: move uniform bank changes to separate PR
asddongmen Jul 22, 2026
52a0082
eventservice: fast-path resolved notifications
asddongmen Jul 25, 2026
b3b74f3
*: merge latest upstream master
asddongmen Jul 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions downstreamadapter/dispatchermanager/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,17 @@ import (
"github.com/pingcap/ticdc/heartbeatpb"
"github.com/pingcap/ticdc/pkg/common"
appcontext "github.com/pingcap/ticdc/pkg/common/context"
"github.com/pingcap/ticdc/pkg/config"
"github.com/pingcap/ticdc/utils/threadpool"
"go.uber.org/zap"
)

const (
defaultHeartbeatInterval = 200 * time.Millisecond
lowLatencyHeartbeatInterval = 50 * time.Millisecond
defaultHeartbeatInitialDelay = time.Second
)

// HeartbeatTask is a perioic task to collect the heartbeat status from event dispatcher manager and push to heartbeatRequestQueue
type HeartBeatTask struct {
taskHandle *threadpool.TaskHandle
Expand All @@ -42,16 +49,15 @@ func newHeartBeatTask(manager *DispatcherManager) *HeartBeatTask {
manager: manager,
statusTick: 0,
}
t.taskHandle = taskScheduler.Submit(t, time.Now().Add(time.Second*1))
t.taskHandle = taskScheduler.Submit(t, time.Now().Add(heartbeatInitialDelay()))
return t
}

func (t *HeartBeatTask) Execute() time.Time {
if t.manager.closed.Load() {
return time.Time{}
}
executeInterval := time.Millisecond * 200
// 10s / 200ms = 50
executeInterval := heartbeatInterval()
completeStatusInterval := int(time.Second * 10 / executeInterval)
t.statusTick++
needCompleteStatus := (t.statusTick)%completeStatusInterval == 0
Expand All @@ -60,6 +66,20 @@ func (t *HeartBeatTask) Execute() time.Time {
return time.Now().Add(executeInterval)
}

func heartbeatInterval() time.Duration {
if config.GetGlobalServerConfig().IsLowLatencyMode() {
return lowLatencyHeartbeatInterval
}
return defaultHeartbeatInterval
}

func heartbeatInitialDelay() time.Duration {
if config.GetGlobalServerConfig().IsLowLatencyMode() {
return 0
}
return defaultHeartbeatInitialDelay
}

func (t *HeartBeatTask) Cancel() {
t.taskHandle.Cancel()
}
Expand Down
38 changes: 38 additions & 0 deletions downstreamadapter/dispatchermanager/task_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright 2026 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package dispatchermanager

import (
"testing"

"github.com/pingcap/ticdc/pkg/config"
"github.com/stretchr/testify/require"
)

func TestHeartbeatIntervalsByPerformanceMode(t *testing.T) {
original := config.GetGlobalServerConfig()
t.Cleanup(func() {
config.StoreGlobalServerConfig(original)
})

cfg := original.Clone()
config.StoreGlobalServerConfig(cfg)
require.Equal(t, defaultHeartbeatInterval, heartbeatInterval())
require.Equal(t, defaultHeartbeatInitialDelay, heartbeatInitialDelay())

cfg.PerformanceMode = config.PerformanceModeLowLatency
config.StoreGlobalServerConfig(cfg)
require.Equal(t, lowLatencyHeartbeatInterval, heartbeatInterval())
require.Zero(t, heartbeatInitialDelay())
}
108 changes: 85 additions & 23 deletions logservice/coordinator/coordinator.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,18 @@ type requestAndTarget struct {
type changefeedState struct {
cfID common.ChangeFeedID
nodeStates map[node.ID]uint64
// nodesReportedSinceLastUpdate tracks a complete reporting round. Publishing
// the global minimum only after every current node has reported avoids
// exposing intermediate minima from staggered node reports.
nodesReportedSinceLastUpdate map[node.ID]struct{}
nodeReportPhyTs map[node.ID]int64

// equal to min puller resolved ts
minLogServiceResolvedTs uint64
resolvedTsGauge prometheus.Gauge
resolvedTsLagGauge prometheus.Gauge
minLogServiceResolvedTs uint64
metricsInitialized bool
metricsUpdatedSinceLastTick bool
resolvedTsGauge prometheus.Gauge
resolvedTsLagGauge prometheus.Gauge
}

type logCoordinator struct {
Expand Down Expand Up @@ -207,6 +214,8 @@ func (c *logCoordinator) handleNodeChange(allNodes map[node.ID]*node.Info) {
c.changefeedStates.Lock()
for _, state := range c.changefeedStates.m {
delete(state.nodeStates, id)
delete(state.nodesReportedSinceLastUpdate, id)
delete(state.nodeReportPhyTs, id)
}
c.changefeedStates.Unlock()
}
Expand All @@ -229,9 +238,11 @@ func (c *logCoordinator) updateEventStoreState(nodeID node.ID, newState *logserv
func (c *logCoordinator) updateChangefeedStates(from node.ID, states *logservicepb.ChangefeedStates) {
c.changefeedStates.Lock()
defer c.changefeedStates.Unlock()
pdPhyTs := oracle.GetPhysical(c.pdClock.CurrentTime())

// Create a set of incoming changefeed GIDs for efficient lookup.
incomingGIDs := make(map[common.GID]struct{})
affectedGIDs := make(map[common.GID]struct{})
for _, state := range states.States {
cfID := common.NewChangefeedIDFromPB(state.GetChangefeedID())
incomingGIDs[cfID.ID()] = struct{}{}
Expand All @@ -244,6 +255,9 @@ func (c *logCoordinator) updateChangefeedStates(from node.ID, states *logservice
// ...but is no longer in the incoming message, it means the changefeed was removed from this node.
if _, incoming := incomingGIDs[gid]; !incoming {
delete(state.nodeStates, from)
delete(state.nodesReportedSinceLastUpdate, from)
delete(state.nodeReportPhyTs, from)
affectedGIDs[gid] = struct{}{}
log.Info("changefeed removed from node",
zap.Stringer("changefeedID", state.cfID),
zap.String("nodeID", string(from)),
Expand Down Expand Up @@ -274,13 +288,34 @@ func (c *logCoordinator) updateChangefeedStates(from node.ID, states *logservice
zap.Uint64("changefeedGIDHigh", gid.High))
// Initialize metrics for the new changefeed.
c.changefeedStates.m[gid] = &changefeedState{
cfID: cfID,
nodeStates: make(map[node.ID]uint64),
resolvedTsGauge: metrics.ChangefeedResolvedTsGauge.WithLabelValues(cfID.Keyspace(), cfID.Name()),
resolvedTsLagGauge: metrics.ChangefeedResolvedTsLagGauge.WithLabelValues(cfID.Keyspace(), cfID.Name()),
cfID: cfID,
nodeStates: make(map[node.ID]uint64),
nodesReportedSinceLastUpdate: make(map[node.ID]struct{}),
nodeReportPhyTs: make(map[node.ID]int64),
resolvedTsGauge: metrics.ChangefeedResolvedTsGauge.WithLabelValues(cfID.Keyspace(), cfID.Name()),
resolvedTsLagGauge: metrics.ChangefeedResolvedTsLagGauge.WithLabelValues(cfID.Keyspace(), cfID.Name()),
}
}
changefeedState := c.changefeedStates.m[gid]
changefeedState.nodeStates[from] = state.GetResolvedTs()
changefeedState.nodesReportedSinceLastUpdate[from] = struct{}{}
changefeedState.nodeReportPhyTs[from] = pdPhyTs
affectedGIDs[gid] = struct{}{}
}

if len(affectedGIDs) > 0 {
for gid := range affectedGIDs {
if state, ok := c.changefeedStates.m[gid]; ok {
if len(state.nodeStates) == 0 ||
len(state.nodesReportedSinceLastUpdate) != len(state.nodeStates) {
continue
}
if c.updateChangefeedMetrics(state, pdPhyTs, false) {
state.metricsUpdatedSinceLastTick = true
}
clear(state.nodesReportedSinceLastUpdate)
}
}
c.changefeedStates.m[gid].nodeStates[from] = state.GetResolvedTs()
}
}

Expand All @@ -292,29 +327,56 @@ func (c *logCoordinator) reportChangefeedMetrics() {
defer c.changefeedStates.Unlock()

for _, state := range c.changefeedStates.m {
if len(state.nodeStates) == 0 {
if state.metricsUpdatedSinceLastTick {
state.metricsUpdatedSinceLastTick = false
continue
}
c.updateChangefeedMetrics(state, pdPhyTs, true)
}
}

minResolvedTs := uint64(math.MaxUint64)
for _, resolvedTs := range state.nodeStates {
if resolvedTs < minResolvedTs {
minResolvedTs = resolvedTs
func (c *logCoordinator) updateChangefeedMetrics(state *changefeedState, pdPhyTs int64, force bool) bool {
if len(state.nodeStates) == 0 {
return false
}

minResolvedTs := uint64(math.MaxUint64)
var maxNodeLag float64
hasNodeLag := false
for nodeID, resolvedTs := range state.nodeStates {
if resolvedTs < minResolvedTs {
minResolvedTs = resolvedTs
}
if !force {
if reportPhyTs, ok := state.nodeReportPhyTs[nodeID]; ok {
nodeLag := float64(reportPhyTs-oracle.ExtractPhysical(resolvedTs)) / 1e3
if !hasNodeLag || nodeLag > maxNodeLag {
maxNodeLag = nodeLag
hasNodeLag = true
}
}
}
}

if minResolvedTs == math.MaxUint64 {
log.Warn("minResolvedTs is MaxUint64, this should not happen",
zap.Stringer("changefeedID", state.cfID))
continue
}
if minResolvedTs == math.MaxUint64 {
log.Warn("minResolvedTs is MaxUint64, this should not happen",
zap.Stringer("changefeedID", state.cfID))
return false
}
if !force && state.metricsInitialized && minResolvedTs == state.minLogServiceResolvedTs {
return false
}

phyResolvedTs := oracle.ExtractPhysical(minResolvedTs)
state.minLogServiceResolvedTs = minResolvedTs
state.resolvedTsGauge.Set(float64(phyResolvedTs))
lag := float64(pdPhyTs-phyResolvedTs) / 1e3
state.resolvedTsLagGauge.Set(lag)
phyResolvedTs := oracle.ExtractPhysical(minResolvedTs)
state.minLogServiceResolvedTs = minResolvedTs
state.metricsInitialized = true
state.resolvedTsGauge.Set(float64(phyResolvedTs))
lag := float64(pdPhyTs-phyResolvedTs) / 1e3
if !force && hasNodeLag {
lag = maxNodeLag
}
state.resolvedTsLagGauge.Set(lag)
return true
}

func (c *logCoordinator) getMinLogServiceResolvedTs(cfID common.ChangeFeedID) uint64 {
Expand Down
Loading
Loading