From 1eec46ae6bacd2f90b8884caf642ead39e22ca4e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 11:30:11 -0700 Subject: [PATCH 01/21] logpuller: extract generic priority queue --- logservice/logpuller/priority_queue.go | 126 ----- logservice/logpuller/priority_queue_test.go | 493 ------------------ logservice/logpuller/priority_task_test.go | 51 ++ logservice/logpuller/subscription_client.go | 15 +- .../logpuller/subscription_client_test.go | 8 +- utils/priorityqueue/priority_queue.go | 149 ++++++ utils/priorityqueue/priority_queue_test.go | 260 +++++++++ 7 files changed, 475 insertions(+), 627 deletions(-) delete mode 100644 logservice/logpuller/priority_queue.go delete mode 100644 logservice/logpuller/priority_queue_test.go create mode 100644 utils/priorityqueue/priority_queue.go create mode 100644 utils/priorityqueue/priority_queue_test.go diff --git a/logservice/logpuller/priority_queue.go b/logservice/logpuller/priority_queue.go deleted file mode 100644 index f19b6d34b9..0000000000 --- a/logservice/logpuller/priority_queue.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync" - - "github.com/pingcap/errors" - "github.com/pingcap/ticdc/utils/heap" -) - -// PriorityQueue is a thread-safe priority queue for region tasks -// It integrates a signal channel to support blocking operations -type PriorityQueue struct { - mu sync.Mutex - heap *heap.Heap[PriorityTask] - - // signal channel for blocking operations - signal chan struct{} -} - -// NewPriorityQueue creates a new priority queue -func NewPriorityQueue() *PriorityQueue { - return &PriorityQueue{ - heap: heap.NewHeap[PriorityTask](), - signal: make(chan struct{}, 1024), - } -} - -// Push adds a task to the priority queue and sends a signal -// This is a non-blocking operation -func (pq *PriorityQueue) Push(task PriorityTask) { - pq.mu.Lock() - pq.heap.AddOrUpdate(task) - pq.mu.Unlock() - - // Send signal to notify waiting consumers - select { - case pq.signal <- struct{}{}: - default: - // Signal channel is full, ignore - } -} - -// Pop removes and returns the highest priority task -// This is a blocking operation that waits for a signal -// Returns nil if the context is cancelled -func (pq *PriorityQueue) Pop(ctx context.Context) (PriorityTask, error) { - for { - // First try to pop without waiting - pq.mu.Lock() - task, ok := pq.heap.PopTop() - pq.mu.Unlock() - - if ok { - return task, nil - } - - // Queue is empty, wait for signal - select { - case <-ctx.Done(): - return nil, ctx.Err() - case _, ok := <-pq.signal: - if !ok { - // Signal channel is closed. - return nil, errors.New("signal channel is closed") - } - // Got signal, try to pop again - continue - } - } -} - -// TryPop attempts to pop a task without blocking -// Returns nil if the queue is empty -func (pq *PriorityQueue) TryPop() PriorityTask { - pq.mu.Lock() - defer pq.mu.Unlock() - - task, ok := pq.heap.PopTop() - if !ok { - return nil - } - return task -} - -// Peek returns the highest priority task without removing it -// Returns nil if the queue is empty -func (pq *PriorityQueue) Peek() PriorityTask { - pq.mu.Lock() - defer pq.mu.Unlock() - - task, ok := pq.heap.PeekTop() - if !ok { - return nil - } - return task -} - -// Len returns the number of tasks in the queue -func (pq *PriorityQueue) Len() int { - pq.mu.Lock() - defer pq.mu.Unlock() - - return pq.heap.Len() -} - -// Close closes the signal channel -func (pq *PriorityQueue) Close() { - // pop all tasks - for pq.Len() > 0 { - pq.TryPop() - } -} diff --git a/logservice/logpuller/priority_queue_test.go b/logservice/logpuller/priority_queue_test.go deleted file mode 100644 index 2bae2456db..0000000000 --- a/logservice/logpuller/priority_queue_test.go +++ /dev/null @@ -1,493 +0,0 @@ -// Copyright 2025 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package logpuller - -import ( - "context" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/pingcap/ticdc/heartbeatpb" - "github.com/stretchr/testify/require" - "github.com/tikv/client-go/v2/oracle" - "github.com/tikv/client-go/v2/tikv" -) - -// mockPriorityTask is a simple mock implementation of PriorityTask for testing -type mockPriorityTask struct { - priority int - heapIndex int - regionInfo regionInfo - description string -} - -func newMockPriorityTask(priority int, description string) *mockPriorityTask { - // Create a minimal regionInfo for testing - verID := tikv.NewRegionVerID(1, 1, 1) - span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} - - // Create a subscribedSpan with atomic resolvedTs - subscribedSpan := &subscribedSpan{ - resolvedTs: atomic.Uint64{}, - } - subscribedSpan.resolvedTs.Store(oracle.GoTimeToTS(time.Now())) - - regionInfo := regionInfo{ - verID: verID, - span: span, - subscribedSpan: subscribedSpan, - } - - return &mockPriorityTask{ - priority: priority, - heapIndex: 0, - regionInfo: regionInfo, - description: description, - } -} - -func (m *mockPriorityTask) Priority() int { - return m.priority -} - -func (m *mockPriorityTask) GetRegionInfo() regionInfo { - return m.regionInfo -} - -func (m *mockPriorityTask) SetHeapIndex(index int) { - m.heapIndex = index -} - -func (m *mockPriorityTask) GetHeapIndex() int { - return m.heapIndex -} - -func (m *mockPriorityTask) LessThan(other PriorityTask) bool { - return m.Priority() < other.Priority() -} - -func TestNewPriorityQueue(t *testing.T) { - pq := NewPriorityQueue() - require.NotNil(t, pq) - require.NotNil(t, pq.heap) - require.NotNil(t, pq.signal) - require.Equal(t, 0, pq.Len()) -} - -func TestPriorityQueue_Push(t *testing.T) { - pq := NewPriorityQueue() - - task1 := newMockPriorityTask(10, "task1") - task2 := newMockPriorityTask(5, "task2") - - // Test pushing single task - pq.Push(task1) - require.Equal(t, 1, pq.Len()) - - // Test pushing multiple tasks - pq.Push(task2) - require.Equal(t, 2, pq.Len()) - - // Verify signal channel receives notifications - select { - case <-pq.signal: - // Expected - signal received - case <-time.After(time.Millisecond * 100): - t.Fatal("Expected signal but none received") - } -} - -func TestPriorityQueue_Peek(t *testing.T) { - pq := NewPriorityQueue() - - // Test peek on empty queue - task := pq.Peek() - require.Nil(t, task) - - // Add tasks with different priorities - task1 := newMockPriorityTask(10, "task1") - task2 := newMockPriorityTask(5, "task2") // Higher priority (lower value) - task3 := newMockPriorityTask(15, "task3") - - pq.Push(task1) - pq.Push(task2) - pq.Push(task3) - - // Peek should return highest priority task (lowest value) - topTask := pq.Peek() - require.NotNil(t, topTask) - require.Equal(t, 5, topTask.Priority()) - require.Equal(t, "task2", topTask.(*mockPriorityTask).description) - - // Verify peek doesn't remove the task - require.Equal(t, 3, pq.Len()) - - // Peek again should return the same task - topTaskAgain := pq.Peek() - require.Equal(t, topTask, topTaskAgain) -} - -func TestPriorityQueue_PopBlocking(t *testing.T) { - pq := NewPriorityQueue() - - // Test pop on empty queue with context cancellation - t.Run("PopWithCancellation", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*50) - defer cancel() - - start := time.Now() - task, err := pq.Pop(ctx) - require.Error(t, err) - elapsed := time.Since(start) - - require.Nil(t, task) - require.True(t, elapsed >= time.Millisecond*50) - }) - - // Test pop with signal - t.Run("PopWithSignal", func(t *testing.T) { - ctx := context.Background() - - // Add a task in a goroutine after a short delay - go func() { - time.Sleep(time.Millisecond * 50) - task1 := newMockPriorityTask(10, "task1") - pq.Push(task1) - }() - - start := time.Now() - task, err := pq.Pop(ctx) - require.NoError(t, err) - elapsed := time.Since(start) - - require.NotNil(t, task) - require.Equal(t, 10, task.Priority()) - require.True(t, elapsed >= time.Millisecond*50) - require.True(t, elapsed < time.Millisecond*200) // Should not wait too long - }) -} - -func TestPriorityQueue_PopOrder(t *testing.T) { - pq := NewPriorityQueue() - ctx := context.Background() - - // Add tasks with different priorities - tasks := []*mockPriorityTask{ - newMockPriorityTask(10, "task1"), - newMockPriorityTask(5, "task2"), // Highest priority - newMockPriorityTask(15, "task3"), - newMockPriorityTask(7, "task4"), - newMockPriorityTask(12, "task5"), - } - - for _, task := range tasks { - pq.Push(task) - } - - // Pop tasks and verify they come out in priority order - expectedOrder := []string{"task2", "task4", "task1", "task5", "task3"} - expectedPriorities := []int{5, 7, 10, 12, 15} - - for i, expectedDesc := range expectedOrder { - task, err := pq.Pop(ctx) - require.NoError(t, err) - require.NotNil(t, task) - require.Equal(t, expectedPriorities[i], task.Priority()) - require.Equal(t, expectedDesc, task.(*mockPriorityTask).description) - } - - // Verify queue is empty - require.Equal(t, 0, pq.Len()) -} - -func TestPriorityQueue_Len(t *testing.T) { - pq := NewPriorityQueue() - - // Test empty queue - require.Equal(t, 0, pq.Len()) - - // Add tasks and verify length - for i := 0; i < 5; i++ { - task := newMockPriorityTask(i, "task") - pq.Push(task) - require.Equal(t, i+1, pq.Len()) - } - - // Remove tasks and verify length - ctx := context.Background() - for i := 4; i >= 0; i-- { - pq.Pop(ctx) - require.Equal(t, i, pq.Len()) - } -} - -func TestPriorityQueue_ConcurrentOperations(t *testing.T) { - pq := NewPriorityQueue() - - numProducers := 3 - numConsumers := 2 - tasksPerProducer := 10 - totalTasks := numProducers * tasksPerProducer - - var wg sync.WaitGroup - var consumedCount int64 - var mu sync.Mutex - consumedTasks := make([]PriorityTask, 0, totalTasks) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Start consumers - for i := 0; i < numConsumers; i++ { - wg.Add(1) - go func(consumerID int) { - defer wg.Done() - for { - task, err := pq.Pop(ctx) - if err != nil { - return - } - - mu.Lock() - consumedTasks = append(consumedTasks, task) - count := atomic.AddInt64(&consumedCount, 1) - mu.Unlock() - - if count >= int64(totalTasks) { - cancel() // Signal other consumers to stop - return - } - } - }(i) - } - - // Start producers - for i := 0; i < numProducers; i++ { - wg.Add(1) - go func(producerID int) { - defer wg.Done() - for j := 0; j < tasksPerProducer; j++ { - priority := (producerID * tasksPerProducer) + j - task := newMockPriorityTask(priority, "concurrent_task") - pq.Push(task) - time.Sleep(time.Microsecond * 10) // Small delay to simulate real work - } - }(i) - } - - // Wait for all producers to finish - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - - // Wait with timeout - select { - case <-done: - // Success - case <-time.After(time.Second * 5): - cancel() // Cancel to stop consumers - t.Fatal("Test timed out") - } - - // Verify all tasks were consumed - require.Equal(t, int64(totalTasks), atomic.LoadInt64(&consumedCount)) - require.Equal(t, totalTasks, len(consumedTasks)) - - // Verify all tasks were processed - for i := 0; i < len(consumedTasks); i++ { - require.NotNil(t, consumedTasks[i]) - } -} - -func TestPriorityQueue_SignalChannelFull(t *testing.T) { - pq := NewPriorityQueue() - - // Fill the signal channel to capacity - for i := 0; i < cap(pq.signal); i++ { - select { - case pq.signal <- struct{}{}: - default: - t.Fatalf("Failed to fill signal channel at iteration %d", i) - } - } - - // Push a task when signal channel is full - should not block - task := newMockPriorityTask(10, "task") - start := time.Now() - pq.Push(task) - elapsed := time.Since(start) - - // Should complete quickly even though signal channel is full - require.True(t, elapsed < time.Millisecond*100) - require.Equal(t, 1, pq.Len()) -} - -func TestPriorityQueue_UpdateExistingTask(t *testing.T) { - pq := NewPriorityQueue() - - // Create a task and add it to queue - task := newMockPriorityTask(10, "task") - pq.Push(task) - require.Equal(t, 1, pq.Len()) - - // Update the task's priority and push again - task.priority = 5 - pq.Push(task) - - // Length should still be 1 (task was updated, not added) - require.Equal(t, 1, pq.Len()) - - // Verify the task has the updated priority - ctx := context.Background() - poppedTask, err := pq.Pop(ctx) - require.NoError(t, err) - require.NotNil(t, poppedTask) - require.Equal(t, 5, poppedTask.Priority()) -} - -func TestPriorityQueue_Close(t *testing.T) { - pq := NewPriorityQueue() - - // Add 3 task before closing - task := newMockPriorityTask(10, "task") - pq.Push(task) - require.Equal(t, 1, pq.Len()) - task2 := newMockPriorityTask(5, "task2") - pq.Push(task2) - require.Equal(t, 2, pq.Len()) - task3 := newMockPriorityTask(15, "task3") - pq.Push(task3) - require.Equal(t, 3, pq.Len()) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - wg := sync.WaitGroup{} - wg.Add(3) - go func() { - defer wg.Done() - for i := 0; i < 1000; i++ { - pq.Push(newMockPriorityTask(i, "task")) - } - }() - - go func() { - defer wg.Done() - defer cancel() - - for i := 0; i < 1000; i++ { - // Test that close doesn't panic - require.NotPanics(t, func() { - pq.Close() - }) - } - }() - - go func() { - defer wg.Done() - for i := 0; i < 1000; i++ { - // Make sure it won't block when the queue is closed - pq.Pop(ctx) - } - }() - - wg.Wait() - require.NotPanics(t, func() { - pq.Close() - }) - // Test that the tasks are popped - require.Equal(t, 0, pq.Len()) -} - -func TestPriorityQueue_EmptyQueueOperations(t *testing.T) { - pq := NewPriorityQueue() - - // Test peek on empty queue - task := pq.Peek() - require.Nil(t, task) - - // Test len on empty queue - require.Equal(t, 0, pq.Len()) - - // Test pop on empty queue with immediate cancellation - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - task2, err := pq.Pop(ctx) - require.Nil(t, task2) - require.Error(t, err) -} - -func TestPriorityQueue_RealPriorityTaskIntegration(t *testing.T) { - pq := NewPriorityQueue() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - currentTs := oracle.GoTimeToTS(time.Now()) - - // Create real priority tasks with different types - verID := tikv.NewRegionVerID(1, 1, 1) - span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} - - subscribedSpan := &subscribedSpan{ - resolvedTs: atomic.Uint64{}, - } - subscribedSpan.resolvedTs.Store(oracle.GoTimeToTS(time.Now().Add(-time.Second))) - - regionInfo := regionInfo{ - verID: verID, - span: span, - subscribedSpan: subscribedSpan, - } - - // Create tasks with different priorities - errorTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs+1) - highTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs) - lowTask := NewRegionPriorityTask(TaskLowPrior, regionInfo, currentTs) - - // Add tasks in non-priority order - pq.Push(lowTask) - pq.Push(errorTask) - pq.Push(highTask) - - require.Equal(t, 3, pq.Len()) - - // Pop tasks and verify they come out in priority order - // TaskRegionError should have highest priority (lowest value) - first, err := pq.Pop(ctx) - require.NoError(t, err) - require.NotNil(t, first) - require.Equal(t, TaskHighPrior, first.(*regionPriorityTask).taskType) - - second, err := pq.Pop(ctx) - require.NoError(t, err) - require.NotNil(t, second) - require.Equal(t, TaskHighPrior, second.(*regionPriorityTask).taskType) - - third, err := pq.Pop(ctx) - require.NoError(t, err) - require.NotNil(t, third) - require.Equal(t, TaskLowPrior, third.(*regionPriorityTask).taskType) - - require.Equal(t, 0, pq.Len()) - - pq.Close() - cancel() - task, err := pq.Pop(ctx) - require.Nil(t, task) - require.Error(t, err) -} diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 5b1b373afb..91c1153dd0 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -14,11 +14,16 @@ package logpuller import ( + "context" + "sync/atomic" "testing" "time" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/oracle" + "github.com/tikv/client-go/v2/tikv" ) // TestPriorityCalculationLogic tests the priority calculation logic in isolation @@ -198,3 +203,49 @@ func TestEdgeCases(t *testing.T) { require.Less(t, priority2, priority1, "wait time longer task priority should be higher") }) } + +func TestRegionPriorityTaskQueueOrder(t *testing.T) { + queue := priorityqueue.New[PriorityTask]() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + currentTs := oracle.GoTimeToTS(time.Now()) + verID := tikv.NewRegionVerID(1, 1, 1) + span := heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("z")} + + subscribedSpan := &subscribedSpan{ + resolvedTs: atomic.Uint64{}, + } + subscribedSpan.resolvedTs.Store(oracle.GoTimeToTS(time.Now().Add(-time.Second))) + + regionInfo := regionInfo{ + verID: verID, + span: span, + subscribedSpan: subscribedSpan, + } + + errorTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs+1) + highTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs) + lowTask := NewRegionPriorityTask(TaskLowPrior, regionInfo, currentTs) + + require.True(t, queue.Push(lowTask)) + require.True(t, queue.Push(errorTask)) + require.True(t, queue.Push(highTask)) + + first, ok, err := queue.Pop(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, TaskHighPrior, first.(*regionPriorityTask).taskType) + + second, ok, err := queue.Pop(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, TaskHighPrior, second.(*regionPriorityTask).taskType) + + third, ok, err := queue.Pop(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, TaskLowPrior, third.(*regionPriorityTask).taskType) + + require.Equal(t, 0, queue.Len()) +} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 0ba3b4be24..f1cb6199e0 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/utils/dynstream" + "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/prometheus/client_golang/prometheus" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/oracle" @@ -212,7 +213,7 @@ type subscriptionClient struct { rangeTaskCh chan rangeTask // regionTaskQueue is used to receive region tasks with priority. // The region will be handled in `handleRegions` goroutine. - regionTaskQueue *PriorityQueue + regionTaskQueue *priorityqueue.PriorityQueue[PriorityTask] // resolveLockTaskCh is used to receive resolve lock tasks. // The tasks will be handled in `handleResolveLockTasks` goroutine. resolveLockTaskCh chan resolveLockTask @@ -241,7 +242,7 @@ func NewSubscriptionClient( credential: credential, rangeTaskCh: make(chan rangeTask, 1024), - regionTaskQueue: NewPriorityQueue(), + regionTaskQueue: priorityqueue.New[PriorityTask](), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), errCache: newErrCache(), @@ -600,10 +601,13 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro default: } // Use blocking Pop to wait for tasks - regionTask, err := s.regionTaskQueue.Pop(ctx) + regionTask, ok, err := s.regionTaskQueue.Pop(ctx) if err != nil { return err } + if !ok { + return nil + } region := regionTask.GetRegionInfo() if region.isStopped() { @@ -619,9 +623,10 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro continue } - region, ok := s.attachRPCContextForRegion(ctx, region) + var rpcCtxOK bool + region, rpcCtxOK = s.attachRPCContextForRegion(ctx, region) // If attachRPCContextForRegion fails, the region will be re-scheduled. - if !ok { + if !rpcCtxOK { continue } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 2e87faa4fc..4bbef8defe 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/utils/dynstream" + "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/pingcap/tidb/pkg/store/mockstore/mockcopr" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" @@ -278,7 +279,7 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), - regionTaskQueue: NewPriorityQueue(), + regionTaskQueue: priorityqueue.New[PriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() @@ -300,8 +301,9 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - task, err := client.regionTaskQueue.Pop(ctx) + task, ok, err := client.regionTaskQueue.Pop(ctx) require.NoError(t, err) + require.True(t, ok) region := task.GetRegionInfo() require.True(t, region.isStopped()) require.True(t, region.filterLoop) @@ -384,7 +386,7 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { client := &subscriptionClient{ ds: &mockDynamicStream{}, - regionTaskQueue: NewPriorityQueue(), + regionTaskQueue: priorityqueue.New[PriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) client.cond = sync.NewCond(&client.mu) diff --git a/utils/priorityqueue/priority_queue.go b/utils/priorityqueue/priority_queue.go new file mode 100644 index 0000000000..a66b346d88 --- /dev/null +++ b/utils/priorityqueue/priority_queue.go @@ -0,0 +1,149 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package priorityqueue provides a thread-safe blocking priority queue. +package priorityqueue + +import ( + "context" + "sync" + + "github.com/pingcap/ticdc/utils/heap" +) + +// PriorityQueue is a thread-safe priority queue based on utils/heap. +// +// The queue uses heap.Item.LessThan to order items. Push and AddOrUpdate both +// use heap.AddOrUpdate semantics: if an item is already queued, its heap +// position is updated instead of inserting a duplicate. If an item's ordering +// fields change while it is queued, callers must call AddOrUpdate again to +// restore heap order. +// +// Pop blocks until an item is available, the context is canceled, or the queue +// is closed. TryPop never blocks: it returns the current top item if one exists, +// or ok=false immediately when the queue is empty. +type PriorityQueue[T heap.Item[T]] struct { + mu sync.Mutex + heap *heap.Heap[T] + notify chan struct{} + closed bool +} + +// New creates an empty priority queue. +func New[T heap.Item[T]]() *PriorityQueue[T] { + return &PriorityQueue[T]{ + heap: heap.NewHeap[T](), + notify: make(chan struct{}, 1), + } +} + +// Push adds or updates an item and wakes one blocked Pop caller. +// It returns false if the queue has been closed. +func (q *PriorityQueue[T]) Push(item T) bool { + return q.AddOrUpdate(item) +} + +// AddOrUpdate adds an item if it is not in the queue, or updates its heap +// position if it is already queued. +func (q *PriorityQueue[T]) AddOrUpdate(item T) bool { + q.mu.Lock() + if q.closed { + q.mu.Unlock() + return false + } + q.heap.AddOrUpdate(item) + q.notifyOneLocked() + q.mu.Unlock() + return true +} + +// Pop blocks until an item is available, the queue is closed, or ctx is done. +// ok is false only when the queue is closed and empty. +func (q *PriorityQueue[T]) Pop(ctx context.Context) (item T, ok bool, err error) { + for { + q.mu.Lock() + item, ok = q.heap.PopTop() + if ok { + if q.heap.Len() > 0 { + q.notifyOneLocked() + } + q.mu.Unlock() + return item, true, nil + } + if q.closed { + q.mu.Unlock() + return item, false, nil + } + q.mu.Unlock() + + select { + case <-ctx.Done(): + return item, false, ctx.Err() + case _, open := <-q.notify: + if !open { + continue + } + } + } +} + +// TryPop removes and returns the top item without blocking. +func (q *PriorityQueue[T]) TryPop() (item T, ok bool) { + q.mu.Lock() + defer q.mu.Unlock() + + item, ok = q.heap.PopTop() + if ok && q.heap.Len() > 0 { + q.notifyOneLocked() + } + return item, ok +} + +// Peek returns the top item without removing it. +func (q *PriorityQueue[T]) Peek() (item T, ok bool) { + q.mu.Lock() + defer q.mu.Unlock() + + return q.heap.PeekTop() +} + +// Len returns the number of queued items. +func (q *PriorityQueue[T]) Len() int { + q.mu.Lock() + defer q.mu.Unlock() + + return q.heap.Len() +} + +// Close prevents future pushes and wakes blocked Pop callers. Items already in +// the queue remain available to Pop. +func (q *PriorityQueue[T]) Close() { + q.mu.Lock() + if q.closed { + q.mu.Unlock() + return + } + q.closed = true + close(q.notify) + q.mu.Unlock() +} + +func (q *PriorityQueue[T]) notifyOneLocked() { + if q.closed { + return + } + select { + case q.notify <- struct{}{}: + default: + } +} diff --git a/utils/priorityqueue/priority_queue_test.go b/utils/priorityqueue/priority_queue_test.go new file mode 100644 index 0000000000..0b4edc0690 --- /dev/null +++ b/utils/priorityqueue/priority_queue_test.go @@ -0,0 +1,260 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package priorityqueue + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type mockItem struct { + priority int + heapIndex int + description string +} + +func newMockItem(priority int, description string) *mockItem { + return &mockItem{priority: priority, description: description} +} + +func (m *mockItem) SetHeapIndex(index int) { + m.heapIndex = index +} + +func (m *mockItem) GetHeapIndex() int { + return m.heapIndex +} + +func (m *mockItem) LessThan(other *mockItem) bool { + return m.priority < other.priority +} + +func TestQueuePushPeekPopOrder(t *testing.T) { + q := New[*mockItem]() + + _, ok := q.Peek() + require.False(t, ok) + + tasks := []*mockItem{ + newMockItem(10, "task1"), + newMockItem(5, "task2"), + newMockItem(15, "task3"), + newMockItem(7, "task4"), + newMockItem(12, "task5"), + } + for _, task := range tasks { + require.True(t, q.Push(task)) + } + require.Equal(t, 5, q.Len()) + + top, ok := q.Peek() + require.True(t, ok) + require.Equal(t, "task2", top.description) + require.Equal(t, 5, q.Len()) + + expectedOrder := []string{"task2", "task4", "task1", "task5", "task3"} + for _, expected := range expectedOrder { + task, ok, err := q.Pop(context.Background()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, expected, task.description) + } + require.Equal(t, 0, q.Len()) +} + +func TestQueuePopBlocking(t *testing.T) { + q := New[*mockItem]() + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + start := time.Now() + task, ok, err := q.Pop(ctx) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.False(t, ok) + require.Nil(t, task) + require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) + + go func() { + time.Sleep(50 * time.Millisecond) + q.Push(newMockItem(10, "task1")) + }() + + start = time.Now() + task, ok, err = q.Pop(context.Background()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, "task1", task.description) + require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) +} + +func TestQueueTryPopAndUpdateExistingItem(t *testing.T) { + q := New[*mockItem]() + + _, ok := q.TryPop() + require.False(t, ok) + + task := newMockItem(10, "task") + require.True(t, q.Push(task)) + task.priority = 5 + require.True(t, q.AddOrUpdate(task)) + require.Equal(t, 1, q.Len()) + + poppedTask, ok := q.TryPop() + require.True(t, ok) + require.Equal(t, 5, poppedTask.priority) + require.Equal(t, 0, poppedTask.heapIndex) +} + +func TestQueueConcurrentOperations(t *testing.T) { + q := New[*mockItem]() + + const ( + numProducers = 3 + numConsumers = 2 + tasksPerProducer = 10 + ) + totalTasks := numProducers * tasksPerProducer + + var wg sync.WaitGroup + var consumedCount int64 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for i := 0; i < numConsumers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + task, ok, err := q.Pop(ctx) + if err != nil || !ok { + return + } + require.NotNil(t, task) + if atomic.AddInt64(&consumedCount, 1) >= int64(totalTasks) { + cancel() + return + } + } + }() + } + + for i := 0; i < numProducers; i++ { + wg.Add(1) + go func(producerID int) { + defer wg.Done() + for j := 0; j < tasksPerProducer; j++ { + priority := producerID*tasksPerProducer + j + require.True(t, q.Push(newMockItem(priority, "task"))) + } + }(i) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + case <-time.After(5 * time.Second): + cancel() + t.Fatal("test timed out") + } + require.Equal(t, int64(totalTasks), consumedCount) +} + +func TestQueueClose(t *testing.T) { + q := New[*mockItem]() + require.True(t, q.Push(newMockItem(10, "task1"))) + require.True(t, q.Push(newMockItem(5, "task2"))) + + q.Close() + require.False(t, q.Push(newMockItem(1, "closed"))) + + task, ok, err := q.Pop(context.Background()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, "task2", task.description) + + task, ok, err = q.Pop(context.Background()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, "task1", task.description) + + task, ok, err = q.Pop(context.Background()) + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, task) + + require.NotPanics(t, q.Close) +} + +func TestQueueCloseWakesBlockedPop(t *testing.T) { + q := New[*mockItem]() + + done := make(chan struct{}) + go func() { + defer close(done) + task, ok, err := q.Pop(context.Background()) + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, task) + }() + + q.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Pop was not woken by Close") + } +} + +func TestQueuePushMultipleItemsWakesMultipleBlockedPop(t *testing.T) { + q := New[*mockItem]() + + const waiters = 2 + ready := make(chan struct{}, waiters) + done := make(chan struct{}, waiters) + for i := 0; i < waiters; i++ { + go func() { + ready <- struct{}{} + task, ok, err := q.Pop(context.Background()) + require.NoError(t, err) + require.True(t, ok) + require.NotNil(t, task) + done <- struct{}{} + }() + } + + for i := 0; i < waiters; i++ { + <-ready + } + require.True(t, q.Push(newMockItem(10, "task1"))) + require.True(t, q.Push(newMockItem(20, "task2"))) + + for i := 0; i < waiters; i++ { + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("blocked Pop was not woken") + } + } +} From 6a277f8b62c77d1c82290e0aac6fc257825ce488 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 11:40:16 -0700 Subject: [PATCH 02/21] small refactor --- logservice/logpuller/priority_task_test.go | 9 ++---- logservice/logpuller/subscription_client.go | 13 ++++---- .../logpuller/subscription_client_test.go | 3 +- utils/priorityqueue/priority_queue.go | 21 +++++++----- utils/priorityqueue/priority_queue_test.go | 32 +++++++------------ 5 files changed, 36 insertions(+), 42 deletions(-) diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index 91c1153dd0..d08855d5cf 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -232,19 +232,16 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { require.True(t, queue.Push(errorTask)) require.True(t, queue.Push(highTask)) - first, ok, err := queue.Pop(ctx) + first, err := queue.Pop(ctx) require.NoError(t, err) - require.True(t, ok) require.Equal(t, TaskHighPrior, first.(*regionPriorityTask).taskType) - second, ok, err := queue.Pop(ctx) + second, err := queue.Pop(ctx) require.NoError(t, err) - require.True(t, ok) require.Equal(t, TaskHighPrior, second.(*regionPriorityTask).taskType) - third, ok, err := queue.Pop(ctx) + third, err := queue.Pop(ctx) require.NoError(t, err) - require.True(t, ok) require.Equal(t, TaskLowPrior, third.(*regionPriorityTask).taskType) require.Equal(t, 0, queue.Len()) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index f1cb6199e0..068aa525d8 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -15,6 +15,7 @@ package logpuller import ( "context" + stderrs "errors" "sync" "sync/atomic" "time" @@ -601,13 +602,13 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro default: } // Use blocking Pop to wait for tasks - regionTask, ok, err := s.regionTaskQueue.Pop(ctx) + regionTask, err := s.regionTaskQueue.Pop(ctx) if err != nil { + if stderrs.Is(err, priorityqueue.ErrClosed) { + return nil + } return err } - if !ok { - return nil - } region := regionTask.GetRegionInfo() if region.isStopped() { @@ -634,7 +635,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro worker := store.getRequestWorker() force := regionTask.Priority() <= forcedPriorityBase - ok, err = worker.add(ctx, region, force) + added, err := worker.add(ctx, region, force) if err != nil { log.Warn("subscription client add region request failed", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -643,7 +644,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro return err } - if !ok { + if !added { s.regionTaskQueue.Push(regionTask) continue } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 4bbef8defe..b502c4d7d1 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -301,9 +301,8 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - task, ok, err := client.regionTaskQueue.Pop(ctx) + task, err := client.regionTaskQueue.Pop(ctx) require.NoError(t, err) - require.True(t, ok) region := task.GetRegionInfo() require.True(t, region.isStopped()) require.True(t, region.filterLoop) diff --git a/utils/priorityqueue/priority_queue.go b/utils/priorityqueue/priority_queue.go index a66b346d88..2e073c2f6e 100644 --- a/utils/priorityqueue/priority_queue.go +++ b/utils/priorityqueue/priority_queue.go @@ -16,11 +16,15 @@ package priorityqueue import ( "context" + "errors" "sync" "github.com/pingcap/ticdc/utils/heap" ) +// ErrClosed is returned by Pop when the queue has been closed and drained. +var ErrClosed = errors.New("priority queue is closed") + // PriorityQueue is a thread-safe priority queue based on utils/heap. // // The queue uses heap.Item.LessThan to order items. Push and AddOrUpdate both @@ -30,8 +34,8 @@ import ( // restore heap order. // // Pop blocks until an item is available, the context is canceled, or the queue -// is closed. TryPop never blocks: it returns the current top item if one exists, -// or ok=false immediately when the queue is empty. +// is closed and drained. TryPop never blocks: it returns the current top item if +// one exists, or ok=false immediately when the queue is empty. type PriorityQueue[T heap.Item[T]] struct { mu sync.Mutex heap *heap.Heap[T] @@ -67,28 +71,29 @@ func (q *PriorityQueue[T]) AddOrUpdate(item T) bool { return true } -// Pop blocks until an item is available, the queue is closed, or ctx is done. -// ok is false only when the queue is closed and empty. -func (q *PriorityQueue[T]) Pop(ctx context.Context) (item T, ok bool, err error) { +// Pop blocks until an item is available, the queue is closed and drained, or ctx +// is done. +func (q *PriorityQueue[T]) Pop(ctx context.Context) (item T, err error) { for { q.mu.Lock() + var ok bool item, ok = q.heap.PopTop() if ok { if q.heap.Len() > 0 { q.notifyOneLocked() } q.mu.Unlock() - return item, true, nil + return item, nil } if q.closed { q.mu.Unlock() - return item, false, nil + return item, ErrClosed } q.mu.Unlock() select { case <-ctx.Done(): - return item, false, ctx.Err() + return item, ctx.Err() case _, open := <-q.notify: if !open { continue diff --git a/utils/priorityqueue/priority_queue_test.go b/utils/priorityqueue/priority_queue_test.go index 0b4edc0690..715daa66bc 100644 --- a/utils/priorityqueue/priority_queue_test.go +++ b/utils/priorityqueue/priority_queue_test.go @@ -70,9 +70,8 @@ func TestQueuePushPeekPopOrder(t *testing.T) { expectedOrder := []string{"task2", "task4", "task1", "task5", "task3"} for _, expected := range expectedOrder { - task, ok, err := q.Pop(context.Background()) + task, err := q.Pop(context.Background()) require.NoError(t, err) - require.True(t, ok) require.Equal(t, expected, task.description) } require.Equal(t, 0, q.Len()) @@ -85,9 +84,8 @@ func TestQueuePopBlocking(t *testing.T) { defer cancel() start := time.Now() - task, ok, err := q.Pop(ctx) + task, err := q.Pop(ctx) require.ErrorIs(t, err, context.DeadlineExceeded) - require.False(t, ok) require.Nil(t, task) require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) @@ -97,9 +95,8 @@ func TestQueuePopBlocking(t *testing.T) { }() start = time.Now() - task, ok, err = q.Pop(context.Background()) + task, err = q.Pop(context.Background()) require.NoError(t, err) - require.True(t, ok) require.Equal(t, "task1", task.description) require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond) } @@ -142,8 +139,8 @@ func TestQueueConcurrentOperations(t *testing.T) { go func() { defer wg.Done() for { - task, ok, err := q.Pop(ctx) - if err != nil || !ok { + task, err := q.Pop(ctx) + if err != nil { return } require.NotNil(t, task) @@ -189,19 +186,16 @@ func TestQueueClose(t *testing.T) { q.Close() require.False(t, q.Push(newMockItem(1, "closed"))) - task, ok, err := q.Pop(context.Background()) + task, err := q.Pop(context.Background()) require.NoError(t, err) - require.True(t, ok) require.Equal(t, "task2", task.description) - task, ok, err = q.Pop(context.Background()) + task, err = q.Pop(context.Background()) require.NoError(t, err) - require.True(t, ok) require.Equal(t, "task1", task.description) - task, ok, err = q.Pop(context.Background()) - require.NoError(t, err) - require.False(t, ok) + task, err = q.Pop(context.Background()) + require.ErrorIs(t, err, ErrClosed) require.Nil(t, task) require.NotPanics(t, q.Close) @@ -213,9 +207,8 @@ func TestQueueCloseWakesBlockedPop(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - task, ok, err := q.Pop(context.Background()) - require.NoError(t, err) - require.False(t, ok) + task, err := q.Pop(context.Background()) + require.ErrorIs(t, err, ErrClosed) require.Nil(t, task) }() @@ -236,9 +229,8 @@ func TestQueuePushMultipleItemsWakesMultipleBlockedPop(t *testing.T) { for i := 0; i < waiters; i++ { go func() { ready <- struct{}{} - task, ok, err := q.Pop(context.Background()) + task, err := q.Pop(context.Background()) require.NoError(t, err) - require.True(t, ok) require.NotNil(t, task) done <- struct{}{} }() From 1cca913743ee40483b5a3fc628be7f6a79367b9b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 11:42:02 -0700 Subject: [PATCH 03/21] small fix --- logservice/logpuller/subscription_client.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 068aa525d8..4d44643552 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -624,10 +624,9 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro continue } - var rpcCtxOK bool - region, rpcCtxOK = s.attachRPCContextForRegion(ctx, region) + region, ok := s.attachRPCContextForRegion(ctx, region) // If attachRPCContextForRegion fails, the region will be re-scheduled. - if !rpcCtxOK { + if !ok { continue } @@ -635,7 +634,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro worker := store.getRequestWorker() force := regionTask.Priority() <= forcedPriorityBase - added, err := worker.add(ctx, region, force) + ok, err = worker.add(ctx, region, force) if err != nil { log.Warn("subscription client add region request failed", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -644,7 +643,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro return err } - if !added { + if !ok { s.regionTaskQueue.Push(regionTask) continue } From c1e89d63be82f5ca2f704eed0b8a5f3020e06280 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 11:47:12 -0700 Subject: [PATCH 04/21] small fix --- logservice/logpuller/subscription_client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 4d44643552..fbcffdc4f1 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -15,7 +15,6 @@ package logpuller import ( "context" - stderrs "errors" "sync" "sync/atomic" "time" @@ -604,7 +603,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro // Use blocking Pop to wait for tasks regionTask, err := s.regionTaskQueue.Pop(ctx) if err != nil { - if stderrs.Is(err, priorityqueue.ErrClosed) { + if errors.Is(err, priorityqueue.ErrClosed) { return nil } return err From cda8c92ff69dfbe38e3f35197f6492d12f159ffd Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 11:48:54 -0700 Subject: [PATCH 05/21] fix --- utils/priorityqueue/priority_queue.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/priorityqueue/priority_queue.go b/utils/priorityqueue/priority_queue.go index 2e073c2f6e..fa42cd9b3d 100644 --- a/utils/priorityqueue/priority_queue.go +++ b/utils/priorityqueue/priority_queue.go @@ -108,9 +108,6 @@ func (q *PriorityQueue[T]) TryPop() (item T, ok bool) { defer q.mu.Unlock() item, ok = q.heap.PopTop() - if ok && q.heap.Len() > 0 { - q.notifyOneLocked() - } return item, ok } From 7bc477ce78d46e9ece096c24672d5b8d8d65d83e Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 13:29:43 -0700 Subject: [PATCH 06/21] small fix --- logservice/logpuller/priority_task_test.go | 4 +--- utils/priorityqueue/priority_queue.go | 2 +- utils/priorityqueue/priority_queue_test.go | 18 +++++++----------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index d08855d5cf..d8b3f26d48 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -14,7 +14,6 @@ package logpuller import ( - "context" "sync/atomic" "testing" "time" @@ -206,8 +205,7 @@ func TestEdgeCases(t *testing.T) { func TestRegionPriorityTaskQueueOrder(t *testing.T) { queue := priorityqueue.New[PriorityTask]() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() currentTs := oracle.GoTimeToTS(time.Now()) verID := tikv.NewRegionVerID(1, 1, 1) diff --git a/utils/priorityqueue/priority_queue.go b/utils/priorityqueue/priority_queue.go index fa42cd9b3d..e6f5fca0c6 100644 --- a/utils/priorityqueue/priority_queue.go +++ b/utils/priorityqueue/priority_queue.go @@ -16,9 +16,9 @@ package priorityqueue import ( "context" - "errors" "sync" + "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/utils/heap" ) diff --git a/utils/priorityqueue/priority_queue_test.go b/utils/priorityqueue/priority_queue_test.go index 715daa66bc..abf29351a7 100644 --- a/utils/priorityqueue/priority_queue_test.go +++ b/utils/priorityqueue/priority_queue_test.go @@ -134,10 +134,8 @@ func TestQueueConcurrentOperations(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - for i := 0; i < numConsumers; i++ { - wg.Add(1) - go func() { - defer wg.Done() + for range numConsumers { + wg.Go(func() { for { task, err := q.Pop(ctx) if err != nil { @@ -149,18 +147,16 @@ func TestQueueConcurrentOperations(t *testing.T) { return } } - }() + }) } - for i := 0; i < numProducers; i++ { - wg.Add(1) - go func(producerID int) { - defer wg.Done() - for j := 0; j < tasksPerProducer; j++ { + for producerID := range numProducers { + wg.Go(func() { + for j := range tasksPerProducer { priority := producerID*tasksPerProducer + j require.True(t, q.Push(newMockItem(priority, "task"))) } - }(i) + }) } done := make(chan struct{}) From c9c9aab3ce4d9932d9a88c7c4715aa3f89cf9b96 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 13:45:30 -0700 Subject: [PATCH 07/21] fix --- utils/priorityqueue/priority_queue_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/priorityqueue/priority_queue_test.go b/utils/priorityqueue/priority_queue_test.go index abf29351a7..7e9dba6d1e 100644 --- a/utils/priorityqueue/priority_queue_test.go +++ b/utils/priorityqueue/priority_queue_test.go @@ -222,7 +222,7 @@ func TestQueuePushMultipleItemsWakesMultipleBlockedPop(t *testing.T) { const waiters = 2 ready := make(chan struct{}, waiters) done := make(chan struct{}, waiters) - for i := 0; i < waiters; i++ { + for range waiters { go func() { ready <- struct{}{} task, err := q.Pop(context.Background()) @@ -232,13 +232,13 @@ func TestQueuePushMultipleItemsWakesMultipleBlockedPop(t *testing.T) { }() } - for i := 0; i < waiters; i++ { + for range waiters { <-ready } require.True(t, q.Push(newMockItem(10, "task1"))) require.True(t, q.Push(newMockItem(20, "task2"))) - for i := 0; i < waiters; i++ { + for range waiters { select { case <-done: case <-time.After(time.Second): From 4e7a5621410017c4ca6391131832289cca74cb6b Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 23 Jun 2026 23:21:21 -0700 Subject: [PATCH 08/21] fix --- logservice/logpuller/priority_task.go | 66 +-- logservice/logpuller/priority_task_test.go | 65 +-- logservice/logpuller/region_req_cache.go | 510 ++++++++++-------- logservice/logpuller/region_req_cache_test.go | 105 +--- logservice/logpuller/region_request_worker.go | 172 ++++-- .../logpuller/region_request_worker_test.go | 24 +- logservice/logpuller/region_state.go | 28 +- logservice/logpuller/subscription_client.go | 171 ++++-- .../logpuller/subscription_client_test.go | 69 ++- 9 files changed, 706 insertions(+), 504 deletions(-) diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/priority_task.go index 68ef9e885d..e173519f72 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/priority_task.go @@ -15,7 +15,7 @@ package logpuller import ( "fmt" - "time" + "sync/atomic" "github.com/tikv/client-go/v2/oracle" ) @@ -42,69 +42,56 @@ func (t TaskType) String() string { return fmt.Sprintf("%d", t) } -// PriorityTask is the interface for priority-based tasks -// It implements heap.Item interface -type PriorityTask interface { - // Priority returns the priority value, lower value means higher priority - Priority() int - - // GetRegionInfo returns the underlying regionInfo - GetRegionInfo() regionInfo - - // heap.Item interface methods - SetHeapIndex(int) - GetHeapIndex() int - LessThan(PriorityTask) bool -} - -// regionPriorityTask implements PriorityTask interface +// regionPriorityTask is a heap item for region scheduling. +// Lower priority values are popped first. The priority value is calculated when +// the task is created, so queued tasks do not rely on heap reordering as time +// passes or resolved-ts changes. type regionPriorityTask struct { taskType TaskType - createTime time.Time + priority int + seq uint64 regionInfo regionInfo heapIndex int // for heap.Item interface - currentTs uint64 + + deferredStore atomic.Pointer[requestedStore] } -// NewRegionPriorityTask creates a new priority task for region -func NewRegionPriorityTask(taskType TaskType, regionInfo regionInfo, currentTs uint64) PriorityTask { +// newRegionPriorityTask creates a new priority task for region. +func newRegionPriorityTask(taskType TaskType, regionInfo regionInfo, currentTs uint64, seq uint64) *regionPriorityTask { return ®ionPriorityTask{ taskType: taskType, - createTime: time.Now(), + priority: calculateRegionTaskPriority(taskType, regionInfo, currentTs), + seq: seq, regionInfo: regionInfo, heapIndex: 0, // 0 means not in heap - currentTs: currentTs, } } -// Priority calculates the priority based on task type and wait time -// Lower value means higher priority -func (pt *regionPriorityTask) Priority() int { - // Base priority based on task type +func calculateRegionTaskPriority(taskType TaskType, regionInfo regionInfo, currentTs uint64) int { basePriority := 0 - switch pt.taskType { + switch taskType { case TaskHighPrior: basePriority = highPriorityBase // Highest priority case TaskLowPrior: basePriority = lowPriorityBase // Lowest priority } - // Add time-based priority bonus - // Wait time in seconds, longer wait time means higher priority (lower value) - waitTime := time.Since(pt.createTime) - timeBonus := int(waitTime.Seconds()) - // ResolvedTsLag in seconds, longer lag means lower priority (higher value) - resolvedTsLag := oracle.GetTimeFromTS(pt.currentTs).Sub(oracle.GetTimeFromTS(pt.regionInfo.subscribedSpan.resolvedTs.Load())) + resolvedTsLag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(regionInfo.subscribedSpan.resolvedTs.Load())) resolvedTsLagPenalty := int(resolvedTsLag.Seconds()) - priority := basePriority - timeBonus + resolvedTsLagPenalty + priority := basePriority + resolvedTsLagPenalty if priority < 0 { priority = 0 } return priority } +// Priority returns the fixed priority value, lower value means higher priority. +func (pt *regionPriorityTask) Priority() int { + return pt.priority +} + // GetRegionInfo returns the underlying regionInfo func (pt *regionPriorityTask) GetRegionInfo() regionInfo { return pt.regionInfo @@ -121,7 +108,10 @@ func (pt *regionPriorityTask) GetHeapIndex() int { } // LessThan implements heap.Item interface -// Returns true if this task has higher priority (lower priority value) than the other task -func (pt *regionPriorityTask) LessThan(other PriorityTask) bool { - return pt.Priority() < other.Priority() +// Returns true if this task has higher priority (lower priority value) than the other task. +func (pt *regionPriorityTask) LessThan(other *regionPriorityTask) bool { + if pt.priority != other.priority { + return pt.priority < other.priority + } + return pt.seq < other.seq } diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/priority_task_test.go index d8b3f26d48..cf6e5123d8 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/priority_task_test.go @@ -35,35 +35,30 @@ func TestPriorityCalculationLogic(t *testing.T) { name string taskType TaskType resolvedTsOffsetSeconds int64 // Offset relative to currentTs (negative means resolvedTs is older) - waitTimeSeconds int // Task wait time description string }{ { name: "high_priority_new_resolvedTs", taskType: TaskHighPrior, resolvedTsOffsetSeconds: -5, // resolvedTs is 5 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds description: "High priority task with newer resolvedTs", }, { name: "high_priority_old_resolvedTs", taskType: TaskHighPrior, resolvedTsOffsetSeconds: -30, // resolvedTs is 30 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds description: "High priority task with older resolvedTs", }, { name: "low_priority_new_resolvedTs", taskType: TaskLowPrior, resolvedTsOffsetSeconds: -5, // resolvedTs is 5 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds description: "Low priority task with newer resolvedTs", }, { name: "low_priority_old_resolvedTs", taskType: TaskLowPrior, resolvedTsOffsetSeconds: -30, // resolvedTs is 30 seconds earlier than currentTs - waitTimeSeconds: 10, // Waited for 10 seconds description: "Low priority task with older resolvedTs", }, } @@ -77,8 +72,7 @@ func TestPriorityCalculationLogic(t *testing.T) { resolvedTime := oracle.GetTimeFromTS(currentTs).Add(time.Duration(tt.resolvedTsOffsetSeconds) * time.Second) resolvedTs := oracle.GoTimeToTS(resolvedTime) - // Simulate priority calculation logic - priority := calculatePriorityDirectly(tt.taskType, currentTs, resolvedTs, tt.waitTimeSeconds) + priority := calculatePriorityDirectly(tt.taskType, currentTs, resolvedTs) t.Logf("%s: Priority = %d", tt.description, priority) priorities = append(priorities, priority) @@ -114,27 +108,20 @@ func TestPriorityCalculationLogic(t *testing.T) { }) } -// calculatePriorityDirectly directly calculates priority for testing -// Copies the logic from regionPriorityTask.Priority() -func calculatePriorityDirectly(taskType TaskType, currentTs, resolvedTs uint64, waitTimeSeconds int) int { - // Base priority based on task type +// calculatePriorityDirectly directly calculates priority for testing. +func calculatePriorityDirectly(taskType TaskType, currentTs, resolvedTs uint64) int { basePriority := 0 switch taskType { case TaskHighPrior: - basePriority = highPriorityBase // 1200 + basePriority = highPriorityBase case TaskLowPrior: - basePriority = lowPriorityBase // 3600 + basePriority = lowPriorityBase } - // Add time-based priority bonus - // Wait time in seconds, longer wait time means higher priority (lower value) - timeBonus := waitTimeSeconds - - // Calculate resolvedTs lag resolvedTsLag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs)) resolvedTsLagBonus := int(resolvedTsLag.Seconds()) - priority := basePriority - timeBonus + resolvedTsLagBonus + priority := basePriority + resolvedTsLagBonus if priority < 0 { priority = 0 @@ -161,8 +148,8 @@ func TestResolvedTsLagLogic(t *testing.T) { require.Less(t, lag2, lag1, "newer resolvedTs should have smaller lag") // Calculate the impact on priority - priority1 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs1, 5) - priority2 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs2, 5) + priority1 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs1) + priority2 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs2) t.Logf("Priority with resolvedTs 10 seconds old: %d", priority1) t.Logf("Priority with resolvedTs 1 second old: %d", priority2) @@ -183,28 +170,28 @@ func TestEdgeCases(t *testing.T) { lag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs)) t.Logf("resolvedTs in the future 5 seconds, lag = %v (%.0f seconds)", lag, lag.Seconds()) - priority := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs, 5) + priority := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs) t.Logf("resolvedTs in the future priority: %d", priority) require.GreaterOrEqual(t, priority, 0, "priority should not be less than 0") }) - t.Run("different wait time impact", func(t *testing.T) { + t.Run("priority is stable after task creation", func(t *testing.T) { resolvedTs := oracle.GoTimeToTS(currentTime.Add(-10 * time.Second)) + subscribedSpan := &subscribedSpan{} + subscribedSpan.resolvedTs.Store(resolvedTs) + regionInfo := regionInfo{subscribedSpan: subscribedSpan} - priority1 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs, 2) - priority2 := calculatePriorityDirectly(TaskHighPrior, currentTs, resolvedTs, 10) - - t.Logf("wait 2 seconds priority: %d", priority1) - t.Logf("wait 10 seconds priority: %d", priority2) + task := newRegionPriorityTask(TaskHighPrior, regionInfo, currentTs, 1) + priority := task.Priority() + subscribedSpan.resolvedTs.Store(currentTs) - // wait time longer task priority should be higher - require.Less(t, priority2, priority1, "wait time longer task priority should be higher") + require.Equal(t, priority, task.Priority()) }) } func TestRegionPriorityTaskQueueOrder(t *testing.T) { - queue := priorityqueue.New[PriorityTask]() + queue := priorityqueue.New[*regionPriorityTask]() ctx := t.Context() currentTs := oracle.GoTimeToTS(time.Now()) @@ -222,25 +209,25 @@ func TestRegionPriorityTaskQueueOrder(t *testing.T) { subscribedSpan: subscribedSpan, } - errorTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs+1) - highTask := NewRegionPriorityTask(TaskHighPrior, regionInfo, currentTs) - lowTask := NewRegionPriorityTask(TaskLowPrior, regionInfo, currentTs) + firstHighTask := newRegionPriorityTask(TaskHighPrior, regionInfo, currentTs, 1) + secondHighTask := newRegionPriorityTask(TaskHighPrior, regionInfo, currentTs, 2) + lowTask := newRegionPriorityTask(TaskLowPrior, regionInfo, currentTs, 3) require.True(t, queue.Push(lowTask)) - require.True(t, queue.Push(errorTask)) - require.True(t, queue.Push(highTask)) + require.True(t, queue.Push(secondHighTask)) + require.True(t, queue.Push(firstHighTask)) first, err := queue.Pop(ctx) require.NoError(t, err) - require.Equal(t, TaskHighPrior, first.(*regionPriorityTask).taskType) + require.Same(t, firstHighTask, first) second, err := queue.Pop(ctx) require.NoError(t, err) - require.Equal(t, TaskHighPrior, second.(*regionPriorityTask).taskType) + require.Same(t, secondHighTask, second) third, err := queue.Pop(ctx) require.NoError(t, err) - require.Equal(t, TaskLowPrior, third.(*regionPriorityTask).taskType) + require.Same(t, lowTask, third) require.Equal(t, 0, queue.Len()) } diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_req_cache.go index b4478cab07..f4ada2c589 100644 --- a/logservice/logpuller/region_req_cache.go +++ b/logservice/logpuller/region_req_cache.go @@ -20,322 +20,396 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" - "go.uber.org/atomic" "go.uber.org/zap" ) const ( - checkStaleRequestInterval = time.Second * 10 - requestGCLifeTime = time.Minute * 180 addReqRetryInterval = time.Millisecond * 1 addReqRetryLimit = 3 abnormalRequestDurationInSec = 60 * 60 * 2 // 2 hours ) -// regionReq represents a wrapped region request with state +type regionReqStage uint8 + +const ( + regionReqStageQueued regionReqStage = iota + regionReqStageProcessing + regionReqStageSent + regionReqStageFinished +) + +type regionReqKey struct { + subID SubscriptionID + regionID uint64 +} + +func newRegionReqKey(region regionInfo) regionReqKey { + return regionReqKey{ + subID: region.subscribedSpan.subID, + regionID: region.verID.GetID(), + } +} + +// regionReq tracks one data request from admission to completion. type regionReq struct { regionInfo regionInfo createTime time.Time + + cache *requestCache + key regionReqKey + stage regionReqStage + + replacesActive bool } -func newRegionReq(region regionInfo) regionReq { - return regionReq{ +func newRegionReq(cache *requestCache, region regionInfo) *regionReq { + return ®ionReq{ regionInfo: region, createTime: time.Now(), + cache: cache, + key: newRegionReqKey(region), + stage: regionReqStageQueued, } } -func (r *regionReq) isStale() bool { - return time.Since(r.createTime) > requestGCLifeTime +func (r *regionReq) markSent() { + if r == nil || r.cache == nil { + return + } + r.cache.markSent(r) } -// requestCache manages region requests with flow control -type requestCache struct { - // pending requests waiting to be sent - pendingQueue chan regionReq +func (r *regionReq) resolve() bool { + if r == nil || r.cache == nil { + return false + } + return r.cache.resolve(r) +} - // sent requests waiting for initialization (subscriptionID -> regions -> regionReq) - sentRequests struct { - sync.RWMutex - regionReqs map[SubscriptionID]map[uint64]regionReq +func (r *regionReq) finish() bool { + if r == nil || r.cache == nil { + return false } + return r.cache.finish(r) +} + +// requestCache manages worker-local data requests with flow control. +// +// requests is the source of truth for live requests. A request is inserted by +// add(), moves through queued/processing/sent, and is removed by resolve(), +// finish(), takeUnsentRegions(), or clear(). +type requestCache struct { + mu sync.Mutex + + requests map[*regionReq]struct{} + current map[regionReqKey]*regionReq - // pendingCount is a flow control slot counter. - // A slot is acquired when a request is successfully enqueued into pendingQueue (see add), - // and is released when the request is finished/removed (resolve/markStopped/markDone/clear). - // pop and markSent don't change it. If markSent overwrites an existing request for the same region, - // it will release a slot for the replaced request to avoid leaking pendingCount. - pendingCount atomic.Int64 - // maximum number of pending requests allowed - maxPendingCount int64 + ready []*regionReq + readyIdx int - // channel to signal when space becomes available - spaceAvailable chan struct{} + maxPendingCount int - lastCheckStaleRequestTime atomic.Time + readyAvailable chan struct{} + spaceAvailable chan struct{} + onSpaceAvailable func() } -func newRequestCache(maxPendingCount int) *requestCache { +func newRequestCache(maxPendingCount int, onSpaceAvailable ...func()) *requestCache { res := &requestCache{ - pendingQueue: make(chan regionReq, maxPendingCount), // Large buffer to reduce blocking - sentRequests: struct { - sync.RWMutex - regionReqs map[SubscriptionID]map[uint64]regionReq - }{regionReqs: make(map[SubscriptionID]map[uint64]regionReq)}, - pendingCount: atomic.Int64{}, - maxPendingCount: int64(maxPendingCount), - spaceAvailable: make(chan struct{}, 16), // Buffered to avoid blocking + requests: make(map[*regionReq]struct{}), + current: make(map[regionReqKey]*regionReq), + ready: make([]*regionReq, 0, maxPendingCount), + maxPendingCount: maxPendingCount, + readyAvailable: make(chan struct{}, 1), + spaceAvailable: make(chan struct{}, 1), + onSpaceAvailable: nil, + } + if len(onSpaceAvailable) > 0 { + res.onSpaceAvailable = onSpaceAvailable[0] } - res.lastCheckStaleRequestTime.Store(time.Now()) return res } -// add adds a new region request to the cache -// It blocks if pendingCount >= maxPendingCount until there's space or ctx is cancelled +// add admits a data request into the worker window. func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) (bool, error) { start := time.Now() ticker := time.NewTicker(addReqRetryInterval) defer ticker.Stop() - addReqRetryLimit := addReqRetryLimit + retries := addReqRetryLimit for { - current := c.pendingCount.Load() - if current < c.maxPendingCount || force { - // Try to add the request - req := newRegionReq(region) - select { - case <-ctx.Done(): - return false, ctx.Err() - case c.pendingQueue <- req: - c.pendingCount.Inc() - cost := time.Since(start) - metrics.SubscriptionClientAddRegionRequestDuration.Observe(cost.Seconds()) - return true, nil - case <-ticker.C: - addReqRetryLimit-- - if addReqRetryLimit <= 0 { - return false, nil - } - continue - } + if c.tryAdd(region, force) { + metrics.SubscriptionClientAddRegionRequestDuration.Observe(time.Since(start).Seconds()) + return true, nil } - // Wait for space to become available select { case <-ticker.C: - addReqRetryLimit-- - if addReqRetryLimit <= 0 { + retries-- + if retries <= 0 { return false, nil } - continue case <-c.spaceAvailable: - continue case <-ctx.Done(): return false, ctx.Err() } } } -// pop gets the next pending request. -// Note: it doesn't change pendingCount. The slot acquired in add() should be released later -// (e.g. resolve/markStopped/markDone). -func (c *requestCache) pop(ctx context.Context) (regionReq, error) { - select { - case req := <-c.pendingQueue: - return req, nil - case <-ctx.Done(): - return regionReq{}, ctx.Err() +func (c *requestCache) tryAdd(region regionInfo, force bool) bool { + req := newRegionReq(c, region) + notifyReady := false + + c.mu.Lock() + defer func() { + c.mu.Unlock() + if notifyReady { + c.notifyReady() + } + }() + + if existing, ok := c.current[req.key]; ok { + if existing.stage == regionReqStageQueued { + existing.regionInfo = region + return true + } + req.replacesActive = true + log.Warn("duplicate active region request detected, keep newest request", + zap.Uint64("subID", uint64(existing.key.subID)), + zap.Uint64("regionID", existing.key.regionID), + zap.Uint8("stage", uint8(existing.stage)), + zap.Int("pendingCount", len(c.requests))) } + if len(c.requests) >= c.maxPendingCount && !force { + return false + } + + c.requests[req] = struct{}{} + c.current[req.key] = req + c.ready = append(c.ready, req) + notifyReady = true + return true } -// markSent marks a request as sent and adds it to sent requests. -// It doesn't change pendingCount: the slot is released when the request is finished/removed. -func (c *requestCache) markSent(req regionReq) { - c.sentRequests.Lock() - defer c.sentRequests.Unlock() +func (c *requestCache) canAdd(force bool) bool { + c.mu.Lock() + defer c.mu.Unlock() - m, ok := c.sentRequests.regionReqs[req.regionInfo.subscribedSpan.subID] + return force || len(c.requests) < c.maxPendingCount +} - if !ok { - m = make(map[uint64]regionReq) - c.sentRequests.regionReqs[req.regionInfo.subscribedSpan.subID] = m - } +// pop takes the next queued request and moves it into processing state. +func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { + for { + if req := c.tryPop(); req != nil { + return req, nil + } - if oldReq, exists := m[req.regionInfo.verID.GetID()]; exists { - log.Warn("region request overwritten", - zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), - zap.Uint64("regionID", req.regionInfo.verID.GetID()), - zap.Float64("oldAgeSec", time.Since(oldReq.createTime).Seconds()), - zap.Float64("newAgeSec", time.Since(req.createTime).Seconds()), - zap.Int("pendingCount", int(c.pendingCount.Load())), - zap.Int("pendingQueueLen", len(c.pendingQueue))) - c.markDone() + select { + case <-c.readyAvailable: + case <-ctx.Done(): + return nil, ctx.Err() + } } - m[req.regionInfo.verID.GetID()] = req } -// markStopped removes a sent request and releases a slot. -func (c *requestCache) markStopped(subID SubscriptionID, regionID uint64) { - c.sentRequests.Lock() - defer c.sentRequests.Unlock() +func (c *requestCache) tryPop() *regionReq { + c.mu.Lock() + defer c.mu.Unlock() - regionReqs, ok := c.sentRequests.regionReqs[subID] - if !ok { - return - } + for c.readyIdx < len(c.ready) { + req := c.ready[c.readyIdx] + c.ready[c.readyIdx] = nil + c.readyIdx++ - _, exists := regionReqs[regionID] - if !exists { - return - } + if req == nil { + continue + } + if _, ok := c.requests[req]; !ok || req.stage != regionReqStageQueued { + continue + } - delete(regionReqs, regionID) - if len(regionReqs) == 0 { - delete(c.sentRequests.regionReqs, subID) + req.stage = regionReqStageProcessing + c.compactReadyLocked() + return req } - c.markDone() + + c.compactReadyLocked() + return nil } -// resolve marks a region as initialized and removes it from sent requests -func (c *requestCache) resolve(subscriptionID SubscriptionID, regionID uint64) bool { - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - regionReqs, ok := c.sentRequests.regionReqs[subscriptionID] - if !ok { - return false +func (c *requestCache) markSent(req *regionReq) { + removed := false + c.mu.Lock() + if _, ok := c.requests[req]; ok && req.stage == regionReqStageProcessing { + if req.replacesActive { + if old := c.findSentLocked(req.key, req); old != nil { + removed = c.removeLocked(old) || removed + } + } + req.stage = regionReqStageSent } + c.mu.Unlock() - req, exists := regionReqs[regionID] - if !exists { + if removed { + c.notifySpace() + } +} + +func (c *requestCache) resolve(req *regionReq) bool { + if !c.remove(req) { return false } - // Check if the subscription ID matches - if req.regionInfo.subscribedSpan.subID == subscriptionID { - delete(regionReqs, regionID) - c.markDone() - cost := time.Since(req.createTime).Seconds() - if cost > 0 && cost < abnormalRequestDurationInSec { - log.Debug("cdc resolve region request", zap.Uint64("subID", uint64(subscriptionID)), zap.Uint64("regionID", regionID), zap.Float64("cost", cost), zap.Int("pendingCount", int(c.pendingCount.Load())), zap.Int("pendingQueueLen", len(c.pendingQueue))) - metrics.RegionRequestFinishScanDuration.Observe(cost) - } else { - log.Info("region request duration abnormal, skip metric", zap.Float64("cost", cost), zap.Uint64("regionID", regionID)) - } + cost := time.Since(req.createTime).Seconds() + if cost > 0 && cost < abnormalRequestDurationInSec { + log.Debug("cdc resolve region request", + zap.Uint64("subID", uint64(req.key.subID)), + zap.Uint64("regionID", req.key.regionID), + zap.Float64("cost", cost), + zap.Int("pendingCount", c.getPendingCount())) + metrics.RegionRequestFinishScanDuration.Observe(cost) return true } + log.Info("region request duration abnormal, skip metric", + zap.Float64("cost", cost), + zap.Uint64("regionID", req.key.regionID)) + return true +} - return false +func (c *requestCache) finish(req *regionReq) bool { + return c.remove(req) } -// clearStaleRequest clears stale requests from the cache -// Note: Sometimes, the CDC sends the same region request to TiKV multiple times. In such cases, this method is needed to reduce the pendingSize. -func (c *requestCache) clearStaleRequest() { - if time.Since(c.lastCheckStaleRequestTime.Load()) < checkStaleRequestInterval { - return +func (c *requestCache) remove(req *regionReq) bool { + if req == nil { + return false } - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - reqCount := 0 - for subID, regionReqs := range c.sentRequests.regionReqs { - for regionID, regionReq := range regionReqs { - if regionReq.regionInfo.isStopped() || - regionReq.regionInfo.subscribedSpan.stopped.Load() || - regionReq.regionInfo.lockedRangeState.Initialized.Load() || - regionReq.isStale() { - c.markDone() - log.Warn("region worker delete stale region request", - zap.Uint64("subID", uint64(subID)), - zap.Uint64("regionID", regionID), - zap.Int("pendingCount", int(c.pendingCount.Load())), - zap.Int("pendingQueueLen", len(c.pendingQueue)), - zap.Bool("isRegionStopped", regionReq.regionInfo.isStopped()), - zap.Bool("isSubscribedSpanStopped", regionReq.regionInfo.subscribedSpan.stopped.Load()), - zap.Bool("isStale", regionReq.isStale()), - zap.Time("createTime", regionReq.createTime)) - delete(regionReqs, regionID) - } else { - reqCount++ - } - } - if len(regionReqs) == 0 { - delete(c.sentRequests.regionReqs, subID) - } + + c.mu.Lock() + removed := c.removeLocked(req) + c.mu.Unlock() + + if removed { + c.notifySpace() } + return removed +} - // If there are no in-cache region requests but pendingCount isn't 0, it means pendingCount is stale. - // Reset it to avoid blocking add() forever. - if reqCount == 0 && len(c.pendingQueue) == 0 && c.pendingCount.Load() != 0 { - log.Info("region worker pending request count is not equal to actual region request count, correct it", - zap.Int("pendingCount", int(c.pendingCount.Load())), - zap.Int("actualReqCount", reqCount), - zap.Int("pendingQueueLen", len(c.pendingQueue))) - c.pendingCount.Store(0) - // Notify waiting add operations that there's space available. - select { - case c.spaceAvailable <- struct{}{}: - default: +func (c *requestCache) takeUnsentRegions() []regionInfo { + c.mu.Lock() + regions := make([]regionInfo, 0, len(c.requests)) + removed := 0 + for req := range c.requests { + if req.stage == regionReqStageSent { + continue } + regions = append(regions, req.regionInfo) + if c.removeLocked(req) { + removed++ + } + } + if removed > 0 { + c.compactReadyLocked() } + c.mu.Unlock() - c.lastCheckStaleRequestTime.Store(time.Now()) + if removed > 0 { + c.notifySpace() + } + return regions } -// clear removes all requests and returns them +// clear removes all live requests and returns their regions. func (c *requestCache) clear() []regionInfo { - var regions []regionInfo - - // Drain pending requests from channel -LOOP: - for { - select { - case req := <-c.pendingQueue: - regions = append(regions, req.regionInfo) - c.markDone() - default: - break LOOP - } + c.mu.Lock() + regions := make([]regionInfo, 0, len(c.requests)) + for req := range c.requests { + regions = append(regions, req.regionInfo) + delete(c.requests, req) + req.stage = regionReqStageFinished } - - c.sentRequests.Lock() - defer c.sentRequests.Unlock() - - for subID, regionReqs := range c.sentRequests.regionReqs { - for regionID := range regionReqs { - regions = append(regions, regionReqs[regionID].regionInfo) - delete(regionReqs, regionID) - c.markDone() - } - delete(c.sentRequests.regionReqs, subID) + removed := len(regions) + c.current = make(map[regionReqKey]*regionReq) + c.ready = c.ready[:0] + c.readyIdx = 0 + c.mu.Unlock() + + if removed > 0 { + c.notifySpace() } return regions } -// getPendingCount returns the current pending count +// getPendingCount returns the number of queued, processing and sent requests. func (c *requestCache) getPendingCount() int { - return int(c.pendingCount.Load()) + c.mu.Lock() + defer c.mu.Unlock() + return len(c.requests) } -func (c *requestCache) markDone() { - // Decrement pendingCount by 1, but never let it go below 0. - // Do it with CAS to avoid clobbering concurrent Inc() calls. - for { - old := c.pendingCount.Load() - if old == 0 { - break - } else if old < 0 { - if c.pendingCount.CompareAndSwap(old, 0) { - break - } - } else { - if c.pendingCount.CompareAndSwap(old, old-1) { - break - } +func (c *requestCache) removeLocked(req *regionReq) bool { + if req == nil { + return false + } + if _, ok := c.requests[req]; !ok { + return false + } + + stage := req.stage + delete(c.requests, req) + req.stage = regionReqStageFinished + if c.current[req.key] == req { + delete(c.current, req.key) + } + if stage != regionReqStageSent { + c.compactReadyLocked() + } + return true +} + +func (c *requestCache) findSentLocked(key regionReqKey, except *regionReq) *regionReq { + for req := range c.requests { + if req == except { + continue + } + if req.key == key && req.stage == regionReqStageSent { + return req } } - // Notify waiting add operations that there's space available. + return nil +} + +func (c *requestCache) notifyReady() { + select { + case c.readyAvailable <- struct{}{}: + default: + } +} + +func (c *requestCache) notifySpace() { select { case c.spaceAvailable <- struct{}{}: - default: // If channel is full, skip notification + default: + } + if c.onSpaceAvailable != nil { + c.onSpaceAvailable() + } +} + +func (c *requestCache) compactReadyLocked() { + if c.readyIdx == 0 { + return + } + if c.readyIdx < len(c.ready) && c.readyIdx < 1024 { + return + } + + n := copy(c.ready, c.ready[c.readyIdx:]) + for i := n; i < len(c.ready); i++ { + c.ready[i] = nil } + c.ready = c.ready[:n] + c.readyIdx = 0 } diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_req_cache_test.go index 62706a8542..428844e9dd 100644 --- a/logservice/logpuller/region_req_cache_test.go +++ b/logservice/logpuller/region_req_cache_test.go @@ -77,30 +77,23 @@ func TestRequestCacheAdd_ForceFlag(t *testing.T) { require.False(t, ok) require.NoError(t, err) - // With force=true, it should still fail because the channel is full - // The force flag only bypasses the pendingCount check, not the channel capacity + // With force=true, the request bypasses the live request limit. region3 := createTestRegionInfo(1, 3) ok, err = cache.add(ctx, region3, true) - require.False(t, ok) + require.True(t, ok) require.NoError(t, err) + require.Equal(t, 2, cache.getPendingCount()) - // consume the pending queue ann add with force req, err := cache.pop(ctx) require.NoError(t, err) require.NotNil(t, req) require.Equal(t, region1.verID.GetID(), req.regionInfo.verID.GetID()) require.Equal(t, region1.subscribedSpan.subID, req.regionInfo.subscribedSpan.subID) - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - - ok, err = cache.add(ctx, region3, true) - require.True(t, ok) - require.NoError(t, err) - // It is 2 since region1 is unresolved require.Equal(t, 2, cache.getPendingCount()) + req.markSent() // resolve region1 - cache.resolve(region1.subscribedSpan.subID, region1.verID.GetID()) + req.resolve() require.Equal(t, 1, cache.getPendingCount()) } @@ -164,12 +157,11 @@ func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { require.NoError(t, err) require.NotNil(t, req) require.Equal(t, 2, cache.getPendingCount()) // pop doesn't change pendingCount - // Mark as sent - cache.markSent(req) + req.markSent() require.Equal(t, 2, cache.getPendingCount()) // Resolve the request to free up space - success := cache.resolve(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + success := req.resolve() require.True(t, success) require.Equal(t, 1, cache.getPendingCount()) @@ -212,75 +204,29 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { require.Equal(t, numGoroutines, cache.getPendingCount()) } -func TestRequestCacheAdd_StaleRequestCleanup(t *testing.T) { +func TestRequestCacheAdd_DuplicateQueuedRequestUpdatesExisting(t *testing.T) { cache := newRequestCache(10) ctx := context.Background() - // Add a request and mark it as sent region := createTestRegionInfo(1, 1) + updatedRegion := region + updatedRegion.filterLoop = true + ok, err := cache.add(ctx, region, false) require.True(t, ok) require.NoError(t, err) - req, err := cache.pop(ctx) - require.NoError(t, err) - require.NotNil(t, req) - - // Mark as sent - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - - // Manually set the request as stale by modifying createTime - cache.sentRequests.Lock() - regionReqs := cache.sentRequests.regionReqs[req.regionInfo.subscribedSpan.subID] - regionReqs[req.regionInfo.verID.GetID()] = regionReq{ - regionInfo: req.regionInfo, - createTime: time.Now().Add(-requestGCLifeTime - time.Second), // Make it stale - } - cache.sentRequests.Unlock() - - // Manually set lastCheckStaleRequestTime to bypass the time interval check - cache.lastCheckStaleRequestTime.Store(time.Now().Add(-checkStaleRequestInterval - time.Second)) - - // Manually trigger stale cleanup by calling clearStaleRequest - cache.clearStaleRequest() - - // The stale request should be cleaned up - require.Equal(t, 0, cache.getPendingCount()) -} - -func TestRequestCacheAdd_WithStoppedRegion(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - // Create a region info with stopped state (lockedRangeState = nil) - region := createTestRegionInfo(1, 1) - region.lockedRangeState = nil // This makes it stopped - - ok, err := cache.add(ctx, region, false) + ok, err = cache.add(ctx, updatedRegion, false) require.True(t, ok) require.NoError(t, err) require.Equal(t, 1, cache.getPendingCount()) req, err := cache.pop(ctx) require.NoError(t, err) - require.NotNil(t, req) - - // Mark as sent - cache.markSent(req) - require.Equal(t, 1, cache.getPendingCount()) - - // Manually set lastCheckStaleRequestTime to bypass the time interval check - cache.lastCheckStaleRequestTime.Store(time.Now().Add(-checkStaleRequestInterval - time.Second)) - - // Manually trigger cleanup of stopped region - cache.clearStaleRequest() - - // The stopped region should be cleaned up - require.Equal(t, 0, cache.getPendingCount()) + require.True(t, req.regionInfo.filterLoop) } -func TestRequestCacheMarkSent_DuplicateReleaseSlot(t *testing.T) { +func TestRequestCacheMarkSent_DuplicateActiveRequestReleasesOldSent(t *testing.T) { cache := newRequestCache(10) ctx := context.Background() @@ -290,29 +236,26 @@ func TestRequestCacheMarkSent_DuplicateReleaseSlot(t *testing.T) { require.True(t, ok) require.NoError(t, err) - // Add a duplicate request for the same region. It should not leak pendingCount even if - // markSent overwrites the existing entry. - ok, err = cache.add(ctx, region, false) - require.True(t, ok) + req1, err := cache.pop(ctx) require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) + req1.markSent() - req1, err := cache.pop(ctx) + ok, err = cache.add(ctx, region, false) + require.True(t, ok) require.NoError(t, err) - cache.markSent(req1) require.Equal(t, 2, cache.getPendingCount()) req2, err := cache.pop(ctx) require.NoError(t, err) - cache.markSent(req2) + req2.markSent() require.Equal(t, 1, cache.getPendingCount()) // Finish the remaining tracked request. - require.True(t, cache.resolve(region.subscribedSpan.subID, region.verID.GetID())) + require.True(t, req2.resolve()) require.Equal(t, 0, cache.getPendingCount()) } -func TestRequestCacheMarkStopped_ReleasesSlot(t *testing.T) { +func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { cache := newRequestCache(10) ctx := context.Background() @@ -326,11 +269,9 @@ func TestRequestCacheMarkStopped_ReleasesSlot(t *testing.T) { req, err := cache.pop(ctx) require.NoError(t, err) - cache.markSent(req) + req.markSent() require.Equal(t, 1, cache.getPendingCount()) - require.Contains(t, cache.sentRequests.regionReqs, req.regionInfo.subscribedSpan.subID) - cache.markStopped(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + req.finish() require.Equal(t, 0, cache.getPendingCount()) - require.NotContains(t, cache.sentRequests.regionReqs, req.regionInfo.subscribedSpan.subID) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 142f4c6493..e2c36e4b21 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/pkg/version" + "github.com/pingcap/ticdc/utils/chann" "go.uber.org/zap" "golang.org/x/sync/errgroup" grpcstatus "google.golang.org/grpc/status" @@ -38,6 +39,42 @@ var workerIDGen atomic.Uint64 type regionFeedStates map[uint64]*regionFeedState +type deregisterRequest struct { + subID SubscriptionID + filterLoop bool +} + +type controlQueue struct { + ch *chann.UnlimitedChannel[deregisterRequest, any] + notify chan struct{} +} + +func newControlQueue() *controlQueue { + return &controlQueue{ + ch: chann.NewUnlimitedChannelDefault[deregisterRequest](), + notify: make(chan struct{}, 1), + } +} + +func (q *controlQueue) push(req deregisterRequest) { + q.ch.Push(req) + select { + case q.notify <- struct{}{}: + default: + } +} + +func (q *controlQueue) tryPop() (deregisterRequest, bool) { + if q.ch.Len() == 0 { + return deregisterRequest{}, false + } + req, ok, err := q.ch.GetWithContext(context.Background()) + if err != nil { + return deregisterRequest{}, false + } + return req, ok +} + // regionRequestWorker is responsible for sending region requests to a specific TiKV store. type regionRequestWorker struct { workerID uint64 @@ -48,10 +85,11 @@ type regionRequestWorker struct { // we must always get a region to request before create a grpc stream. // only in this way we can avoid to try to connect to an offline store infinitely. - preFetchForConnecting *regionInfo + preFetchForConnecting *regionReq // request cache with flow control requestCache *requestCache + controlQueue *controlQueue // all regions maintained by this worker. requestedRegions struct { @@ -70,10 +108,13 @@ func newRegionRequestWorker( requestCacheSize int, ) *regionRequestWorker { worker := ®ionRequestWorker{ - workerID: workerIDGen.Add(1), - client: client, - store: store, - requestCache: newRequestCache(requestCacheSize), + workerID: workerIDGen.Add(1), + client: client, + store: store, + requestCache: newRequestCache(requestCacheSize, func() { + store.promoteDeferredTask(client.regionTaskQueue) + }), + controlQueue: newControlQueue(), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) @@ -89,11 +130,10 @@ func newRegionRequestWorker( return err } if req.regionInfo.isStopped() { - worker.requestCache.markDone() + req.finish() continue } - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo + worker.preFetchForConnecting = req return nil } } @@ -349,6 +389,10 @@ func (s *regionRequestWorker) processRegionSendTask( ctx context.Context, conn *ConnAndClient, ) error { + if s.controlQueue == nil { + s.controlQueue = newControlQueue() + } + doSend := func(req *cdcpb.ChangeDataRequest) error { if err := conn.Client.Send(req); err != nil { log.Warn("region request worker send request to grpc stream failed", @@ -362,13 +406,61 @@ func (s *regionRequestWorker) processRegionSendTask( // TODO: add a metric? return nil } + sendDeregister := func(req deregisterRequest) error { + changeDataReq := &cdcpb.ChangeDataRequest{ + Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + RequestId: uint64(req.subID), + Request: &cdcpb.ChangeDataRequest_Deregister_{ + Deregister: &cdcpb.ChangeDataRequest_Deregister{}, + }, + FilterLoop: req.filterLoop, + } + if err := doSend(changeDataReq); err != nil { + return err + } + for _, state := range s.takeRegionStates(req.subID) { + state.markStopped(&requestCancelledErr{}) + regionEvent := regionEvent{ + states: []*regionFeedState{state}, + } + s.client.pushRegionEventToDS(req.subID, regionEvent) + } + return nil + } + drainControl := func() error { + for { + req, ok := s.controlQueue.tryPop() + if !ok { + return nil + } + if err := sendDeregister(req); err != nil { + return err + } + } + } // Handle pre-fetched region first - region := *s.preFetchForConnecting + regionReq := s.preFetchForConnecting s.preFetchForConnecting = nil - regionReq := newRegionReq(region) - var err error for { + if err := drainControl(); err != nil { + return err + } + if regionReq == nil { + if regionReq = s.requestCache.tryPop(); regionReq != nil { + continue + } + select { + case <-s.controlQueue.notify: + continue + case <-s.requestCache.readyAvailable: + regionReq = s.requestCache.tryPop() + continue + case <-ctx.Done(): + return ctx.Err() + } + } + region := regionReq.regionInfo subID := region.subscribedSpan.subID log.Debug("region request worker gets a singleRegionInfo", @@ -378,35 +470,16 @@ func (s *regionRequestWorker) processRegionSendTask( zap.String("addr", s.store.storeAddr), zap.Bool("bdrMode", region.filterLoop)) - // It means it's a special task for stopping the table. if region.isStopped() { - req := &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, - RequestId: uint64(subID), - Request: &cdcpb.ChangeDataRequest_Deregister_{ - Deregister: &cdcpb.ChangeDataRequest_Deregister{}, - }, - FilterLoop: region.filterLoop, - } - s.requestCache.markDone() - if err := doSend(req); err != nil { - return err - } - for _, state := range s.takeRegionStates(subID) { - state.markStopped(&requestCancelledErr{}) - regionEvent := regionEvent{ - states: []*regionFeedState{state}, - } - s.client.pushRegionEventToDS(subID, regionEvent) - } + regionReq.finish() } else if region.subscribedSpan.stopped.Load() { // It can be skipped directly because there must be no pending states from // the stopped subscribedTable, or the special singleRegionInfo for stopping // the table will be handled later. s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) - s.requestCache.markDone() + regionReq.finish() } else { - state := newRegionFeedState(region, uint64(subID), s) + state := newRegionFeedState(region, uint64(subID), s, regionReq) state.start() s.addRegionState(subID, region.verID.GetID(), state) // Mark the request as sent before sending it. @@ -414,26 +487,21 @@ func (s *regionRequestWorker) processRegionSendTask( // 1. addRegionState makes the region visible to error handling. // 2. doSend sends the request. // 3. the receiver goroutine may receive a region error immediately. - // 4. markStopped runs before markSent, so requestCache.markStopped cannot - // find the request in sentRequests. - // 5. the sender goroutine then calls markSent and leaves a stale sent - // request behind, even though the region has already been - // unlocked/rescheduled. + // 4. markStopped runs before markSent, so the request may be finished + // before it is marked as sent. + // 5. the sender goroutine then calls markSent and must not make the + // finished request live again. // // Tracking the request before Send keeps requestedRegions and - // sentRequests visible in the same order and avoids leaving stale - // requests in cleanup. - s.requestCache.markSent(regionReq) + // request lifecycle visible in the same order and avoids leaving stale + // requests behind. + regionReq.markSent() if err := doSend(s.createRegionRequest(region)); err != nil { state.markStopped(err) return err } } - // Try to get from cache - regionReq, err = s.requestCache.pop(ctx) - if err != nil { - return err - } + regionReq = nil } } @@ -513,16 +581,12 @@ func (s *regionRequestWorker) clearPendingRegions() []regionInfo { // Clear pre-fetched region if s.preFetchForConnecting != nil { - region := *s.preFetchForConnecting + req := s.preFetchForConnecting s.preFetchForConnecting = nil - regions = append(regions, region) - // The pre-fetched region was popped from pendingQueue but hasn't been marked as sent or done yet. - // Release its pendingCount slot to avoid leaking flow control credits on worker failures. - s.requestCache.markDone() + regions = append(regions, req.regionInfo) + req.finish() } - // Clear all regions from cache - cacheRegions := s.requestCache.clear() - regions = append(regions, cacheRegions...) + regions = append(regions, s.requestCache.takeUnsentRegions()...) return regions } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index f9752a0eb3..eb094a8f23 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -93,8 +93,7 @@ func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, worker.requestCache.getPendingCount()) - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo + worker.preFetchForConnecting = req regions := worker.clearPendingRegions() require.Len(t, regions, 1) @@ -285,16 +284,15 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { req, err := worker.requestCache.pop(ctx) require.NoError(t, err) - state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker) + state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) state.start() worker.addRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) - // Simulate the race we are fixing in processRegionSendTask: - // once a request is visible in sentRequests, a fast region error may mark the - // region stopped before worker cleanup runs. In that case, markStopped should - // remove the sent request immediately, so clearPendingRegions must not return - // the stale region again during worker shutdown. - worker.requestCache.markSent(req) + // Simulate the race we are fixing in processRegionSendTask: once a request is + // visible as sent, a fast region error may mark the region stopped before + // worker cleanup runs. In that case, markStopped should finish the request + // immediately, so clearPendingRegions must not return the stale region again. + req.markSent() state.markStopped(errors.New("send request to store error")) worker.takeRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) @@ -320,8 +318,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { req, err := worker.requestCache.pop(ctx) require.NoError(t, err) - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo + worker.preFetchForConnecting = req sendErr := errors.New("send failed") conn := &ConnAndClient{ @@ -332,7 +329,6 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { err = worker.processRegionSendTask(ctx, conn) require.ErrorIs(t, err, sendErr) require.Equal(t, 0, worker.requestCache.getPendingCount()) - require.Empty(t, worker.requestCache.sentRequests.regionReqs) state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") } @@ -370,8 +366,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { req, err := worker.requestCache.pop(ctx) require.NoError(t, err) - worker.preFetchForConnecting = new(regionInfo) - *worker.preFetchForConnecting = req.regionInfo + worker.preFetchForConnecting = req conn := &ConnAndClient{ Client: &mockEventFeedV2Client{sendErr: tc.sendErr}, @@ -382,7 +377,6 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { var streamErr *storeStreamErr require.ErrorAs(t, err, &streamErr) require.Equal(t, 0, worker.requestCache.getPendingCount()) - require.Empty(t, worker.requestCache.sentRequests.regionReqs) state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index e9c21a7aad..9e157bd5d9 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -103,15 +103,25 @@ type regionFeedState struct { err error } - worker *regionRequestWorker + worker *regionRequestWorker + request *regionReq } -func newRegionFeedState(region regionInfo, requestID uint64, worker *regionRequestWorker) *regionFeedState { - return ®ionFeedState{ +func newRegionFeedState( + region regionInfo, + requestID uint64, + worker *regionRequestWorker, + request ...*regionReq, +) *regionFeedState { + state := ®ionFeedState{ region: region, requestID: requestID, worker: worker, } + if len(request) > 0 { + state.request = request[0] + } + return state } func (s *regionFeedState) start() { @@ -126,7 +136,9 @@ func (s *regionFeedState) markStopped(err error) { s.state.v = stateStopped s.state.err = err } - s.worker.requestCache.markStopped(s.region.subscribedSpan.subID, s.region.verID.GetID()) + if s.request != nil { + s.request.finish() + } } // mark regionFeedState as removed if possible. @@ -138,7 +150,9 @@ func (s *regionFeedState) markRemoved() (changed bool) { changed = true s.matcher.clear() } - s.worker.requestCache.markStopped(s.region.subscribedSpan.subID, s.region.verID.GetID()) + if s.request != nil { + s.request.finish() + } return } @@ -162,7 +176,9 @@ func (s *regionFeedState) isInitialized() bool { func (s *regionFeedState) setInitialized() { s.region.lockedRangeState.Initialized.Store(true) - s.worker.requestCache.resolve(s.region.subscribedSpan.subID, s.region.verID.GetID()) + if s.request != nil { + s.request.resolve() + } } func (s *regionFeedState) getRegionID() uint64 { diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index fbcffdc4f1..6b9082f571 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -213,7 +213,8 @@ type subscriptionClient struct { rangeTaskCh chan rangeTask // regionTaskQueue is used to receive region tasks with priority. // The region will be handled in `handleRegions` goroutine. - regionTaskQueue *priorityqueue.PriorityQueue[PriorityTask] + regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] + regionTaskSeq atomic.Uint64 // resolveLockTaskCh is used to receive resolve lock tasks. // The tasks will be handled in `handleResolveLockTasks` goroutine. resolveLockTaskCh chan resolveLockTask @@ -242,7 +243,7 @@ func NewSubscriptionClient( credential: credential, rangeTaskCh: make(chan rangeTask, 1024), - regionTaskQueue: priorityqueue.New[PriorityTask](), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), errCache: newErrCache(), @@ -323,7 +324,6 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { store := value.(*requestedStore) store.requestWorkers.RLock() for _, worker := range store.requestWorkers.s { - worker.requestCache.clearStaleRequest() pendingRegionReqCount += worker.requestCache.getPendingCount() } store.requestWorkers.RUnlock() @@ -493,7 +493,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Then send a special singleRegionInfo to regionRouter to deregister the table // from all TiKV instances. if rt.stopped.CompareAndSwap(false, true) { - s.regionTaskQueue.Push(NewRegionPriorityTask(TaskHighPrior, regionInfo{subscribedSpan: rt, filterLoop: rt.filterLoop}, s.pdClock.CurrentTS())) + s.enqueueDeregisterToAllStores(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -537,6 +537,12 @@ type requestedStore struct { sync.RWMutex s []*regionRequestWorker } + + deferredTasks struct { + sync.Mutex + tasks []*regionPriorityTask + promoted *regionPriorityTask + } } func (rs *requestedStore) getRequestWorker() *regionRequestWorker { @@ -547,6 +553,108 @@ func (rs *requestedStore) getRequestWorker() *regionRequestWorker { return rs.requestWorkers.s[index] } +func (rs *requestedStore) addRegion( + ctx context.Context, region regionInfo, force bool, +) (bool, *regionRequestWorker, error) { + rs.requestWorkers.RLock() + workers := rs.requestWorkers.s + rs.requestWorkers.RUnlock() + + if len(workers) == 0 { + return false, nil, nil + } + start := int(rs.nextWorker.Add(1)) % len(workers) + for i := range len(workers) { + worker := workers[(start+i)%len(workers)] + ok, err := worker.add(ctx, region, force) + if err != nil || ok { + return ok, worker, err + } + } + return false, nil, nil +} + +func (rs *requestedStore) canAdmitTask(task *regionPriorityTask) bool { + rs.deferredTasks.Lock() + defer rs.deferredTasks.Unlock() + + if rs.deferredTasks.promoted == task { + return true + } + return rs.deferredTasks.promoted == nil && len(rs.deferredTasks.tasks) == 0 +} + +func (rs *requestedStore) deferTask(task *regionPriorityTask) { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted == task { + rs.deferredTasks.promoted = nil + task.deferredStore.Store(nil) + rs.deferredTasks.tasks = append([]*regionPriorityTask{task}, rs.deferredTasks.tasks...) + } else { + rs.deferredTasks.tasks = append(rs.deferredTasks.tasks, task) + } + rs.deferredTasks.Unlock() +} + +func (rs *requestedStore) maybePromoteDeferredTask( + regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask], +) { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { + rs.deferredTasks.Unlock() + return + } + force := rs.deferredTasks.tasks[0].Priority() <= forcedPriorityBase + rs.deferredTasks.Unlock() + + if rs.hasRequestCapacity(force) { + rs.promoteDeferredTask(regionTaskQueue) + } +} + +func (rs *requestedStore) hasRequestCapacity(force bool) bool { + rs.requestWorkers.RLock() + workers := rs.requestWorkers.s + rs.requestWorkers.RUnlock() + + for _, worker := range workers { + if worker.requestCache.canAdd(force) { + return true + } + } + return false +} + +func (rs *requestedStore) finishPromotedTask(task *regionPriorityTask) { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted == task { + rs.deferredTasks.promoted = nil + task.deferredStore.Store(nil) + } + rs.deferredTasks.Unlock() +} + +func (rs *requestedStore) promoteDeferredTask(regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask]) { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { + rs.deferredTasks.Unlock() + return + } + task := rs.deferredTasks.tasks[0] + copy(rs.deferredTasks.tasks, rs.deferredTasks.tasks[1:]) + rs.deferredTasks.tasks[len(rs.deferredTasks.tasks)-1] = nil + rs.deferredTasks.tasks = rs.deferredTasks.tasks[:len(rs.deferredTasks.tasks)-1] + rs.deferredTasks.promoted = task + task.deferredStore.Store(rs) + rs.deferredTasks.Unlock() + + regionTaskQueue.Push(task) +} + +func (s *subscriptionClient) newRegionPriorityTask(taskType TaskType, regionInfo regionInfo) *regionPriorityTask { + return newRegionPriorityTask(taskType, regionInfo, s.pdClock.CurrentTS(), s.regionTaskSeq.Add(1)) +} + // handleRegions receives regionInfo from regionTaskQueue and attach rpcCtx to them, // then send them to corresponding requestedStore. func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Group) error { @@ -611,29 +719,35 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro region := regionTask.GetRegionInfo() if region.isStopped() { - enqueued, err := s.enqueueRegionToAllStores(ctx, region) - if err != nil { - return err - } - if !enqueued { - log.Debug("enqueue stop request failed, retry later", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID))) - s.regionTaskQueue.Push(regionTask) - } + s.enqueueDeregisterToAllStores(region.subscribedSpan.subID, region.filterLoop) continue } + promotedStore := regionTask.deferredStore.Load() region, ok := s.attachRPCContextForRegion(ctx, region) // If attachRPCContextForRegion fails, the region will be re-scheduled. if !ok { + if promotedStore != nil { + promotedStore.finishPromotedTask(regionTask) + promotedStore.promoteDeferredTask(s.regionTaskQueue) + } continue } store := getStore(region.rpcCtx.Addr) - worker := store.getRequestWorker() + if promotedStore != nil && promotedStore != store { + promotedStore.finishPromotedTask(regionTask) + promotedStore.promoteDeferredTask(s.regionTaskQueue) + } + if !store.canAdmitTask(regionTask) { + store.deferTask(regionTask) + store.maybePromoteDeferredTask(s.regionTaskQueue) + continue + } force := regionTask.Priority() <= forcedPriorityBase - ok, err = worker.add(ctx, region, force) + var worker *regionRequestWorker + ok, worker, err = store.addRegion(ctx, region, force) if err != nil { log.Warn("subscription client add region request failed", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), @@ -643,9 +757,11 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } if !ok { - s.regionTaskQueue.Push(regionTask) + store.deferTask(regionTask) + store.maybePromoteDeferredTask(s.regionTaskQueue) continue } + store.finishPromotedTask(regionTask) log.Debug("subscription client will request a region", zap.Uint64("workID", worker.workerID), @@ -655,30 +771,23 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro } } -func (s *subscriptionClient) enqueueRegionToAllStores(ctx context.Context, region regionInfo) (bool, error) { - enqueued := true - var firstErr error +func (s *subscriptionClient) enqueueDeregisterToAllStores(subID SubscriptionID, filterLoop bool) { s.stores.Range(func(_ any, value any) bool { rs := value.(*requestedStore) rs.requestWorkers.RLock() workers := rs.requestWorkers.s rs.requestWorkers.RUnlock() for _, worker := range workers { - ok, err := worker.add(ctx, region, true) - if err != nil { - firstErr = err - enqueued = false - return false - } - if !ok { - enqueued = false - // It is likely the store is busy, no need to try other workers in this store now. - break + if worker.controlQueue == nil { + worker.controlQueue = newControlQueue() } + worker.controlQueue.push(deregisterRequest{ + subID: subID, + filterLoop: filterLoop, + }) } return true }) - return enqueued, firstErr } func (s *subscriptionClient) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { @@ -813,7 +922,7 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState - s.regionTaskQueue.Push(NewRegionPriorityTask(priority, region, s.pdClock.CurrentTS())) + s.regionTaskQueue.Push(s.newRegionPriorityTask(priority, region)) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, priority) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index b502c4d7d1..ac98796426 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -279,7 +279,7 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), - regionTaskQueue: priorityqueue.New[PriorityTask](), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() @@ -297,15 +297,17 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) + worker := ®ionRequestWorker{controlQueue: newControlQueue()} + store := &requestedStore{storeAddr: "store-1"} + store.requestWorkers.s = []*regionRequestWorker{worker} + client.stores.Store(store.storeAddr, store) + client.setTableStopped(span) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - task, err := client.regionTaskQueue.Pop(ctx) - require.NoError(t, err) - region := task.GetRegionInfo() - require.True(t, region.isStopped()) - require.True(t, region.filterLoop) + req, ok := worker.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req.subID) + require.True(t, req.filterLoop) } func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { @@ -385,7 +387,7 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { client := &subscriptionClient{ ds: &mockDynamicStream{}, - regionTaskQueue: priorityqueue.New[PriorityTask](), + regionTaskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) client.cond = sync.NewCond(&client.mu) @@ -413,12 +415,13 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { } } -func TestEnqueueRegionToAllStoresRetryWhenCacheFull(t *testing.T) { +func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { ctx := context.Background() client := &subscriptionClient{} worker := ®ionRequestWorker{ requestCache: newRequestCache(1), + controlQueue: newControlQueue(), } store := &requestedStore{storeAddr: "store-1"} store.requestWorkers.s = []*regionRequestWorker{worker} @@ -432,20 +435,44 @@ func TestEnqueueRegionToAllStoresRetryWhenCacheFull(t *testing.T) { require.NoError(t, err) require.True(t, ok) - stopRegion := regionInfo{ - subscribedSpan: &subscribedSpan{subID: SubscriptionID(1)}, - } - enqueued, err := client.enqueueRegionToAllStores(ctx, stopRegion) - require.NoError(t, err) - require.False(t, enqueued) + client.enqueueDeregisterToAllStores(SubscriptionID(1), true) + require.Equal(t, 1, worker.controlQueue.ch.Len()) + req, ok := worker.controlQueue.tryPop() + require.True(t, ok) + require.Equal(t, SubscriptionID(1), req.subID) + require.True(t, req.filterLoop) + require.Equal(t, 1, worker.requestCache.getPendingCount()) +} + +func TestRequestedStoreDeferredTasksBlockSameStore(t *testing.T) { + queue := priorityqueue.New[*regionPriorityTask]() + store := &requestedStore{storeAddr: "store-1"} + currentTs := oracle.GoTimeToTS(time.Now()) + span := &subscribedSpan{subID: SubscriptionID(1)} + span.resolvedTs.Store(currentTs) + region := regionInfo{subscribedSpan: span} - <-worker.requestCache.pendingQueue - worker.requestCache.markDone() + firstTask := newRegionPriorityTask(TaskHighPrior, region, currentTs, 1) + secondTask := newRegionPriorityTask(TaskHighPrior, region, currentTs, 2) - enqueued, err = client.enqueueRegionToAllStores(ctx, stopRegion) + store.deferTask(firstTask) + require.False(t, store.canAdmitTask(secondTask)) + + store.promoteDeferredTask(queue) + promoted, err := queue.Pop(t.Context()) + require.NoError(t, err) + require.Same(t, firstTask, promoted) + require.True(t, store.canAdmitTask(firstTask)) + require.False(t, store.canAdmitTask(secondTask)) + + store.deferTask(firstTask) + store.promoteDeferredTask(queue) + promoted, err = queue.Pop(t.Context()) require.NoError(t, err) - require.True(t, enqueued) - require.Equal(t, 1, len(worker.requestCache.pendingQueue)) + require.Same(t, firstTask, promoted) + + store.finishPromotedTask(firstTask) + require.True(t, store.canAdmitTask(secondTask)) } func TestSubscriptionWithFailedTiKV(t *testing.T) { From 68d67cf250ff52391a6e5451fa1c24fbd0a7d85f Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 24 Jun 2026 23:16:20 -0700 Subject: [PATCH 09/21] fix --- logservice/logpuller/region_req_cache.go | 102 ++++++------------ logservice/logpuller/region_req_cache_test.go | 37 +------ logservice/logpuller/region_request_worker.go | 9 ++ .../logpuller/region_request_worker_test.go | 31 ++++++ 4 files changed, 77 insertions(+), 102 deletions(-) diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_req_cache.go index f4ada2c589..af6fd125f4 100644 --- a/logservice/logpuller/region_req_cache.go +++ b/logservice/logpuller/region_req_cache.go @@ -32,34 +32,29 @@ const ( type regionReqStage uint8 const ( + // regionReqStageQueued means the request has been admitted but not yet + // picked up by the worker send loop. regionReqStageQueued regionReqStage = iota + // regionReqStageProcessing means the send loop has popped the request, but + // it has not been recorded as sent yet. regionReqStageProcessing + // regionReqStageSent means the request has been sent to TiKV and is waiting + // for initialized/resolved/stopped cleanup. regionReqStageSent + // regionReqStageFinished means the request has left the worker window. regionReqStageFinished ) -type regionReqKey struct { - subID SubscriptionID - regionID uint64 -} - -func newRegionReqKey(region regionInfo) regionReqKey { - return regionReqKey{ - subID: region.subscribedSpan.subID, - regionID: region.verID.GetID(), - } -} - // regionReq tracks one data request from admission to completion. type regionReq struct { regionInfo regionInfo createTime time.Time + // cache is set by requestCache.add. It lets regionFeedState finish exactly + // the request that created it instead of looking up by subID/regionID. cache *requestCache - key regionReqKey + // stage is guarded by requestCache.mu. stage regionReqStage - - replacesActive bool } func newRegionReq(cache *requestCache, region regionInfo) *regionReq { @@ -67,7 +62,6 @@ func newRegionReq(cache *requestCache, region regionInfo) *regionReq { regionInfo: region, createTime: time.Now(), cache: cache, - key: newRegionReqKey(region), stage: regionReqStageQueued, } } @@ -101,28 +95,37 @@ func (r *regionReq) finish() bool { type requestCache struct { mu sync.Mutex + // requests owns every live data request in this worker. Its length is the + // flow-control count used by add(), getPendingCount(), and metrics. requests map[*regionReq]struct{} - current map[regionReqKey]*regionReq + // ready is the FIFO list of queued requests. Entries already popped or + // removed are left as nil/stale and skipped by tryPop; readyIdx is the next + // candidate index. compactReadyLocked occasionally drops skipped entries. ready []*regionReq readyIdx int + // maxPendingCount limits len(requests) for non-force adds. force adds bypass + // this limit, matching the old force behavior for high-priority data requests. maxPendingCount int - readyAvailable chan struct{} - spaceAvailable chan struct{} + // readyAvailable wakes a worker blocked in pop() when a queued request is + // appended. It is a level-trigger hint; callers must re-check ready under mu. + readyAvailable chan struct{} + // spaceAvailable wakes add() when a live request leaves requests. + spaceAvailable chan struct{} + // onSpaceAvailable lets the store-level deferred scheduler retry a task when + // this worker frees request capacity. onSpaceAvailable func() } func newRequestCache(maxPendingCount int, onSpaceAvailable ...func()) *requestCache { res := &requestCache{ - requests: make(map[*regionReq]struct{}), - current: make(map[regionReqKey]*regionReq), - ready: make([]*regionReq, 0, maxPendingCount), - maxPendingCount: maxPendingCount, - readyAvailable: make(chan struct{}, 1), - spaceAvailable: make(chan struct{}, 1), - onSpaceAvailable: nil, + requests: make(map[*regionReq]struct{}), + ready: make([]*regionReq, 0, maxPendingCount), + maxPendingCount: maxPendingCount, + readyAvailable: make(chan struct{}, 1), + spaceAvailable: make(chan struct{}, 1), } if len(onSpaceAvailable) > 0 { res.onSpaceAvailable = onSpaceAvailable[0] @@ -158,7 +161,6 @@ func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) ( } func (c *requestCache) tryAdd(region regionInfo, force bool) bool { - req := newRegionReq(c, region) notifyReady := false c.mu.Lock() @@ -169,24 +171,12 @@ func (c *requestCache) tryAdd(region regionInfo, force bool) bool { } }() - if existing, ok := c.current[req.key]; ok { - if existing.stage == regionReqStageQueued { - existing.regionInfo = region - return true - } - req.replacesActive = true - log.Warn("duplicate active region request detected, keep newest request", - zap.Uint64("subID", uint64(existing.key.subID)), - zap.Uint64("regionID", existing.key.regionID), - zap.Uint8("stage", uint8(existing.stage)), - zap.Int("pendingCount", len(c.requests))) - } if len(c.requests) >= c.maxPendingCount && !force { return false } + req := newRegionReq(c, region) c.requests[req] = struct{}{} - c.current[req.key] = req c.ready = append(c.ready, req) notifyReady = true return true @@ -240,21 +230,11 @@ func (c *requestCache) tryPop() *regionReq { } func (c *requestCache) markSent(req *regionReq) { - removed := false c.mu.Lock() if _, ok := c.requests[req]; ok && req.stage == regionReqStageProcessing { - if req.replacesActive { - if old := c.findSentLocked(req.key, req); old != nil { - removed = c.removeLocked(old) || removed - } - } req.stage = regionReqStageSent } c.mu.Unlock() - - if removed { - c.notifySpace() - } } func (c *requestCache) resolve(req *regionReq) bool { @@ -265,8 +245,8 @@ func (c *requestCache) resolve(req *regionReq) bool { cost := time.Since(req.createTime).Seconds() if cost > 0 && cost < abnormalRequestDurationInSec { log.Debug("cdc resolve region request", - zap.Uint64("subID", uint64(req.key.subID)), - zap.Uint64("regionID", req.key.regionID), + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID()), zap.Float64("cost", cost), zap.Int("pendingCount", c.getPendingCount())) metrics.RegionRequestFinishScanDuration.Observe(cost) @@ -274,7 +254,7 @@ func (c *requestCache) resolve(req *regionReq) bool { } log.Info("region request duration abnormal, skip metric", zap.Float64("cost", cost), - zap.Uint64("regionID", req.key.regionID)) + zap.Uint64("regionID", req.regionInfo.verID.GetID())) return true } @@ -331,7 +311,6 @@ func (c *requestCache) clear() []regionInfo { req.stage = regionReqStageFinished } removed := len(regions) - c.current = make(map[regionReqKey]*regionReq) c.ready = c.ready[:0] c.readyIdx = 0 c.mu.Unlock() @@ -360,27 +339,12 @@ func (c *requestCache) removeLocked(req *regionReq) bool { stage := req.stage delete(c.requests, req) req.stage = regionReqStageFinished - if c.current[req.key] == req { - delete(c.current, req.key) - } if stage != regionReqStageSent { c.compactReadyLocked() } return true } -func (c *requestCache) findSentLocked(key regionReqKey, except *regionReq) *regionReq { - for req := range c.requests { - if req == except { - continue - } - if req.key == key && req.stage == regionReqStageSent { - return req - } - } - return nil -} - func (c *requestCache) notifyReady() { select { case c.readyAvailable <- struct{}{}: diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_req_cache_test.go index 428844e9dd..a39bb7d297 100644 --- a/logservice/logpuller/region_req_cache_test.go +++ b/logservice/logpuller/region_req_cache_test.go @@ -204,55 +204,26 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { require.Equal(t, numGoroutines, cache.getPendingCount()) } -func TestRequestCacheAdd_DuplicateQueuedRequestUpdatesExisting(t *testing.T) { +func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testing.T) { cache := newRequestCache(10) ctx := context.Background() region := createTestRegionInfo(1, 1) - updatedRegion := region - updatedRegion.filterLoop = true ok, err := cache.add(ctx, region, false) require.True(t, ok) require.NoError(t, err) - ok, err = cache.add(ctx, updatedRegion, false) - require.True(t, ok) - require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) - - req, err := cache.pop(ctx) - require.NoError(t, err) - require.True(t, req.regionInfo.filterLoop) -} - -func TestRequestCacheMarkSent_DuplicateActiveRequestReleasesOldSent(t *testing.T) { - cache := newRequestCache(10) - ctx := context.Background() - - region := createTestRegionInfo(1, 1) - - ok, err := cache.add(ctx, region, false) - require.True(t, ok) - require.NoError(t, err) - - req1, err := cache.pop(ctx) - require.NoError(t, err) - req1.markSent() - ok, err = cache.add(ctx, region, false) require.True(t, ok) require.NoError(t, err) require.Equal(t, 2, cache.getPendingCount()) + req1, err := cache.pop(ctx) + require.NoError(t, err) req2, err := cache.pop(ctx) require.NoError(t, err) - req2.markSent() - require.Equal(t, 1, cache.getPendingCount()) - - // Finish the remaining tracked request. - require.True(t, req2.resolve()) - require.Equal(t, 0, cache.getPendingCount()) + require.NotSame(t, req1, req2) } func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index e2c36e4b21..72cf1d0e93 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -528,6 +528,15 @@ func (s *regionRequestWorker) addRegionState(subscriptionID SubscriptionID, regi s.requestedRegions.subscriptions[subscriptionID] = states } + if oldState := states[regionID]; oldState != nil && oldState.request != state.request { + log.Warn("region request state overwritten", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", regionID)) + if oldState.request != nil { + oldState.request.finish() + } + } states[regionID] = state } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index eb094a8f23..2d329540d2 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -77,6 +77,37 @@ func TestRegionStatesOperation(t *testing.T) { require.Equal(t, 0, len(worker.requestedRegions.subscriptions)) } +func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { + worker := ®ionRequestWorker{ + requestCache: newRequestCache(10), + } + worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) + + ctx := context.Background() + region := createTestRegionInfo(1, 1) + + ok, err := worker.requestCache.add(ctx, region, false) + require.NoError(t, err) + require.True(t, ok) + req1, err := worker.requestCache.pop(ctx) + require.NoError(t, err) + req1.markSent() + state1 := newRegionFeedState(req1.regionInfo, uint64(req1.regionInfo.subscribedSpan.subID), worker, req1) + worker.addRegionState(req1.regionInfo.subscribedSpan.subID, req1.regionInfo.verID.GetID(), state1) + + ok, err = worker.requestCache.add(ctx, region, false) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 2, worker.requestCache.getPendingCount()) + req2, err := worker.requestCache.pop(ctx) + require.NoError(t, err) + state2 := newRegionFeedState(req2.regionInfo, uint64(req2.regionInfo.subscribedSpan.subID), worker, req2) + worker.addRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID(), state2) + + require.Equal(t, 1, worker.requestCache.getPendingCount()) + require.Same(t, state2, worker.getRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID())) +} + func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), From f73faab078464602f22893fa3213cf411c9fec90 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 25 Jun 2026 10:24:41 -0700 Subject: [PATCH 10/21] small fix --- logservice/logpuller/subscription_client.go | 58 +++++-------------- .../logpuller/subscription_client_test.go | 4 +- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 6b9082f571..f100612d41 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -322,11 +322,9 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { pendingRegionReqCount := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) - store.requestWorkers.RLock() - for _, worker := range store.requestWorkers.s { + for _, worker := range store.requestWorkers { pendingRegionReqCount += worker.requestCache.getPendingCount() } - store.requestWorkers.RUnlock() return true }) @@ -530,13 +528,11 @@ func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { // requestedStore represents a store that has been connected. type requestedStore struct { storeAddr string - // Use to select a worker to send request. + // nextWorker is the round‑robin cursor used to select the next worker to attempt. nextWorker atomic.Uint32 - - requestWorkers struct { - sync.RWMutex - s []*regionRequestWorker - } + // requestWorkers are fully created before requestedStore is published + // and remain immutable afterwards. + requestWorkers []*regionRequestWorker deferredTasks struct { sync.Mutex @@ -545,24 +541,11 @@ type requestedStore struct { } } -func (rs *requestedStore) getRequestWorker() *regionRequestWorker { - rs.requestWorkers.RLock() - defer rs.requestWorkers.RUnlock() - - index := rs.nextWorker.Add(1) % uint32(len(rs.requestWorkers.s)) - return rs.requestWorkers.s[index] -} - func (rs *requestedStore) addRegion( ctx context.Context, region regionInfo, force bool, ) (bool, *regionRequestWorker, error) { - rs.requestWorkers.RLock() - workers := rs.requestWorkers.s - rs.requestWorkers.RUnlock() + workers := rs.requestWorkers - if len(workers) == 0 { - return false, nil, nil - } start := int(rs.nextWorker.Add(1)) % len(workers) for i := range len(workers) { worker := workers[(start+i)%len(workers)] @@ -613,11 +596,7 @@ func (rs *requestedStore) maybePromoteDeferredTask( } func (rs *requestedStore) hasRequestCapacity(force bool) bool { - rs.requestWorkers.RLock() - workers := rs.requestWorkers.s - rs.requestWorkers.RUnlock() - - for _, worker := range workers { + for _, worker := range rs.requestWorkers { if worker.requestCache.canAdd(force) { return true } @@ -660,6 +639,7 @@ func (s *subscriptionClient) newRegionPriorityTask(taskType TaskType, regionInfo func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Group) error { cfg := config.GetGlobalServerConfig() pendingRegionRequestQueueSize := cfg.Debug.Puller.PendingRegionRequestQueueSize + // Store creation is serialized by the single handleRegions loop. getStore := func(storeAddr string) *requestedStore { var rs *requestedStore if v, ok := s.stores.Load(storeAddr); ok { @@ -667,10 +647,6 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro return rs } - rs = &requestedStore{storeAddr: storeAddr} - rs.requestWorkers.s = make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore) - s.stores.Store(storeAddr, rs) - perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.config.RegionRequestWorkerPerStore) if perWorkerQueueSize <= 0 { log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", @@ -679,12 +655,15 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro perWorkerQueueSize = 1 } - rs.requestWorkers.Lock() + rs = &requestedStore{ + storeAddr: storeAddr, + requestWorkers: make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore), + } for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { requestWorker := newRegionRequestWorker(ctx, s, s.credential, eg, rs, perWorkerQueueSize) - rs.requestWorkers.s = append(rs.requestWorkers.s, requestWorker) + rs.requestWorkers = append(rs.requestWorkers, requestWorker) } - rs.requestWorkers.Unlock() + s.stores.Store(storeAddr, rs) return rs } @@ -692,11 +671,9 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro s.stores.Range(func(_, value any) bool { rs := value.(*requestedStore) - rs.requestWorkers.RLock() - for _, w := range rs.requestWorkers.s { + for _, w := range rs.requestWorkers { w.requestCache.clear() } - rs.requestWorkers.RUnlock() return true }) @@ -774,10 +751,7 @@ func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Gro func (s *subscriptionClient) enqueueDeregisterToAllStores(subID SubscriptionID, filterLoop bool) { s.stores.Range(func(_ any, value any) bool { rs := value.(*requestedStore) - rs.requestWorkers.RLock() - workers := rs.requestWorkers.s - rs.requestWorkers.RUnlock() - for _, worker := range workers { + for _, worker := range rs.requestWorkers { if worker.controlQueue == nil { worker.controlQueue = newControlQueue() } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index ac98796426..0b56f05caa 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -299,7 +299,7 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { worker := ®ionRequestWorker{controlQueue: newControlQueue()} store := &requestedStore{storeAddr: "store-1"} - store.requestWorkers.s = []*regionRequestWorker{worker} + store.requestWorkers = []*regionRequestWorker{worker} client.stores.Store(store.storeAddr, store) client.setTableStopped(span) @@ -424,7 +424,7 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { controlQueue: newControlQueue(), } store := &requestedStore{storeAddr: "store-1"} - store.requestWorkers.s = []*regionRequestWorker{worker} + store.requestWorkers = []*regionRequestWorker{worker} client.stores.Store(store.storeAddr, store) dummyRegion := regionInfo{ From f1d83f7f8b94125f203c10ce1c81bd438aa78efc Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 25 Jun 2026 15:29:13 -0700 Subject: [PATCH 11/21] add region request scheduler --- ...iority_task.go => region_priority_task.go} | 9 +- ...k_test.go => region_priority_task_test.go} | 7 +- ...n_req_cache.go => region_request_cache.go} | 0 ...e_test.go => region_request_cache_test.go} | 4 +- .../logpuller/region_request_scheduler.go | 309 ++++++++++++++++++ logservice/logpuller/region_request_worker.go | 10 +- .../logpuller/region_request_worker_test.go | 2 + logservice/logpuller/subscription_client.go | 279 +--------------- .../logpuller/subscription_client_test.go | 32 +- 9 files changed, 353 insertions(+), 299 deletions(-) rename logservice/logpuller/{priority_task.go => region_priority_task.go} (92%) rename logservice/logpuller/{priority_task_test.go => region_priority_task_test.go} (98%) rename logservice/logpuller/{region_req_cache.go => region_request_cache.go} (100%) rename logservice/logpuller/{region_req_cache_test.go => region_request_cache_test.go} (98%) create mode 100644 logservice/logpuller/region_request_scheduler.go diff --git a/logservice/logpuller/priority_task.go b/logservice/logpuller/region_priority_task.go similarity index 92% rename from logservice/logpuller/priority_task.go rename to logservice/logpuller/region_priority_task.go index e173519f72..8a6d8e645d 100644 --- a/logservice/logpuller/priority_task.go +++ b/logservice/logpuller/region_priority_task.go @@ -53,6 +53,9 @@ type regionPriorityTask struct { regionInfo regionInfo heapIndex int // for heap.Item interface + // deferredStore is set when a task deferred by one store is promoted back + // to the global priority queue. It lets the scheduler clear the store's + // promoted marker after the task is admitted, moved, or dropped. deferredStore atomic.Pointer[requestedStore] } @@ -80,11 +83,7 @@ func calculateRegionTaskPriority(taskType TaskType, regionInfo regionInfo, curre resolvedTsLag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(regionInfo.subscribedSpan.resolvedTs.Load())) resolvedTsLagPenalty := int(resolvedTsLag.Seconds()) - priority := basePriority + resolvedTsLagPenalty - if priority < 0 { - priority = 0 - } - return priority + return max(basePriority+resolvedTsLagPenalty, 0) } // Priority returns the fixed priority value, lower value means higher priority. diff --git a/logservice/logpuller/priority_task_test.go b/logservice/logpuller/region_priority_task_test.go similarity index 98% rename from logservice/logpuller/priority_task_test.go rename to logservice/logpuller/region_priority_task_test.go index cf6e5123d8..226d12b216 100644 --- a/logservice/logpuller/priority_task_test.go +++ b/logservice/logpuller/region_priority_task_test.go @@ -121,12 +121,7 @@ func calculatePriorityDirectly(taskType TaskType, currentTs, resolvedTs uint64) resolvedTsLag := oracle.GetTimeFromTS(currentTs).Sub(oracle.GetTimeFromTS(resolvedTs)) resolvedTsLagBonus := int(resolvedTsLag.Seconds()) - priority := basePriority + resolvedTsLagBonus - - if priority < 0 { - priority = 0 - } - return priority + return max(basePriority+resolvedTsLagBonus, 0) } func TestResolvedTsLagLogic(t *testing.T) { diff --git a/logservice/logpuller/region_req_cache.go b/logservice/logpuller/region_request_cache.go similarity index 100% rename from logservice/logpuller/region_req_cache.go rename to logservice/logpuller/region_request_cache.go diff --git a/logservice/logpuller/region_req_cache_test.go b/logservice/logpuller/region_request_cache_test.go similarity index 98% rename from logservice/logpuller/region_req_cache_test.go rename to logservice/logpuller/region_request_cache_test.go index a39bb7d297..6f96a7e138 100644 --- a/logservice/logpuller/region_req_cache_test.go +++ b/logservice/logpuller/region_request_cache_test.go @@ -181,7 +181,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { done := make(chan error, numGoroutines) // Start multiple goroutines adding requests concurrently - for i := 0; i < numGoroutines; i++ { + for i := range numGoroutines { go func(id int) { region := createTestRegionInfo(SubscriptionID(id%3), uint64(id)) ok, err := cache.add(ctx, region, false) @@ -192,7 +192,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { } // Wait for all goroutines to complete - for i := 0; i < numGoroutines; i++ { + for range numGoroutines { select { case err := <-done: require.NoError(t, err) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go new file mode 100644 index 0000000000..2c8e739392 --- /dev/null +++ b/logservice/logpuller/region_request_scheduler.go @@ -0,0 +1,309 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/config" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/utils/priorityqueue" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +// regionRequestScheduler owns region request admission from the global +// priority queue to per-store request workers. +type regionRequestScheduler struct { + client *subscriptionClient + + queue *priorityqueue.PriorityQueue[*regionPriorityTask] + seq atomic.Uint64 + + stores sync.Map +} + +func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { + return ®ionRequestScheduler{ + client: client, + queue: priorityqueue.New[*regionPriorityTask](), + } +} + +func (s *regionRequestScheduler) submit(taskType TaskType, region regionInfo) { + task := newRegionPriorityTask(taskType, region, s.client.pdClock.CurrentTS(), s.seq.Add(1)) + s.queue.Push(task) +} + +func (s *regionRequestScheduler) close() { + s.queue.Close() + s.releaseAdmittedRegionRequests() +} + +func (s *regionRequestScheduler) pendingRequestCount() int { + count := 0 + s.stores.Range(func(_, value any) bool { + store := value.(*requestedStore) + for _, worker := range store.requestWorkers { + count += worker.requestCache.getPendingCount() + } + return true + }) + return count +} + +func (s *regionRequestScheduler) broadcastDeregister(subID SubscriptionID, filterLoop bool) { + s.stores.Range(func(_ any, value any) bool { + rs := value.(*requestedStore) + for _, worker := range rs.requestWorkers { + worker.controlQueue.push(deregisterRequest{ + subID: subID, + filterLoop: filterLoop, + }) + } + return true + }) +} + +// releaseAdmittedRegionRequests releases region requests admitted to store +// workers during scheduler shutdown. +func (s *regionRequestScheduler) releaseAdmittedRegionRequests() { + s.stores.Range(func(_, value any) bool { + rs := value.(*requestedStore) + rs.releaseAdmittedRegionRequests() + return true + }) +} + +// Run receives region tasks from the global priority queue and admits them to +// the corresponding TiKV store worker. +func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) error { + cfg := config.GetGlobalServerConfig() + pendingRegionRequestQueueSize := cfg.Debug.Puller.PendingRegionRequestQueueSize + // Store creation is serialized by the single scheduler loop. + getStore := func(storeAddr string) *requestedStore { + var rs *requestedStore + if v, ok := s.stores.Load(storeAddr); ok { + rs = v.(*requestedStore) + return rs + } + + perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.client.config.RegionRequestWorkerPerStore) + if perWorkerQueueSize <= 0 { + log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", + zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), + zap.Uint("regionRequestWorkerPerStore", s.client.config.RegionRequestWorkerPerStore)) + perWorkerQueueSize = 1 + } + + rs = &requestedStore{ + scheduler: s, + storeAddr: storeAddr, + requestWorkers: make([]*regionRequestWorker, 0, s.client.config.RegionRequestWorkerPerStore), + } + for i := uint(0); i < s.client.config.RegionRequestWorkerPerStore; i++ { + requestWorker := newRegionRequestWorker(ctx, s.client, s.client.credential, eg, rs, perWorkerQueueSize) + rs.requestWorkers = append(rs.requestWorkers, requestWorker) + } + s.stores.Store(storeAddr, rs) + return rs + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + regionTask, err := s.queue.Pop(ctx) + if err != nil { + if errors.Is(err, priorityqueue.ErrClosed) { + return nil + } + return err + } + + region := regionTask.GetRegionInfo() + if region.isStopped() { + s.broadcastDeregister(region.subscribedSpan.subID, region.filterLoop) + continue + } + + promotedStore := regionTask.deferredStore.Load() + region, ok := s.client.attachRPCContextForRegion(ctx, region) + if !ok { + if promotedStore != nil { + promotedStore.finishPromotedTask(regionTask) + promotedStore.promoteDeferredTask() + } + continue + } + + store := getStore(region.rpcCtx.Addr) + if promotedStore != nil && promotedStore != store { + promotedStore.finishPromotedTask(regionTask) + promotedStore.promoteDeferredTask() + } + if store.hasDeferredTaskAhead(regionTask) { + store.deferTask(regionTask) + store.maybePromoteDeferredTask() + continue + } + force := regionTask.Priority() <= forcedPriorityBase + + var worker *regionRequestWorker + ok, worker, err = store.addRegion(ctx, region, force) + if err != nil { + log.Warn("subscription client add region request failed", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.Error(err)) + return err + } + + if !ok { + store.deferTask(regionTask) + store.maybePromoteDeferredTask() + continue + } + store.finishPromotedTask(regionTask) + + log.Debug("subscription client will request a region", + zap.Uint64("workID", worker.workerID), + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", store.storeAddr)) + } +} + +// requestedStore is the scheduler's local state for one TiKV store. +type requestedStore struct { + scheduler *regionRequestScheduler + + storeAddr string + // nextWorker is the round-robin cursor used to select the next worker to attempt. + nextWorker atomic.Uint32 + // requestWorkers are fully created before requestedStore is published + // and remain immutable afterwards. + requestWorkers []*regionRequestWorker + + deferredTasks struct { + sync.Mutex + // tasks are blocked because this store currently has no worker capacity. + tasks []*regionPriorityTask + // promoted is the deferred task that has been pushed back to the global + // queue but has not been admitted by this store yet. + promoted *regionPriorityTask + } +} + +func (rs *requestedStore) addRegion( + ctx context.Context, region regionInfo, force bool, +) (bool, *regionRequestWorker, error) { + workers := rs.requestWorkers + + start := int(rs.nextWorker.Add(1)) % len(workers) + for i := range len(workers) { + worker := workers[(start+i)%len(workers)] + ok, err := worker.add(ctx, region, force) + if err != nil || ok { + return ok, worker, err + } + } + return false, nil, nil +} + +// hasDeferredTaskAhead reports whether an earlier deferred task should be +// retried before task for this store. +func (rs *requestedStore) hasDeferredTaskAhead(task *regionPriorityTask) bool { + rs.deferredTasks.Lock() + defer rs.deferredTasks.Unlock() + + if rs.deferredTasks.promoted == task { + // The promoted task was already removed from the deferred queue and is + // being retried through the global priority queue. + return false + } + return rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) > 0 +} + +func (rs *requestedStore) deferTask(task *regionPriorityTask) { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted == task { + rs.deferredTasks.promoted = nil + task.deferredStore.Store(nil) + rs.deferredTasks.tasks = append([]*regionPriorityTask{task}, rs.deferredTasks.tasks...) + } else { + rs.deferredTasks.tasks = append(rs.deferredTasks.tasks, task) + } + rs.deferredTasks.Unlock() +} + +func (rs *requestedStore) maybePromoteDeferredTask() { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { + rs.deferredTasks.Unlock() + return + } + force := rs.deferredTasks.tasks[0].Priority() <= forcedPriorityBase + rs.deferredTasks.Unlock() + + if rs.hasRequestCapacity(force) { + rs.promoteDeferredTask() + } +} + +func (rs *requestedStore) hasRequestCapacity(force bool) bool { + for _, worker := range rs.requestWorkers { + if worker.requestCache.canAdd(force) { + return true + } + } + return false +} + +func (rs *requestedStore) releaseAdmittedRegionRequests() { + for _, worker := range rs.requestWorkers { + worker.releaseAdmittedRegionRequests() + } +} + +func (rs *requestedStore) finishPromotedTask(task *regionPriorityTask) { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted == task { + rs.deferredTasks.promoted = nil + task.deferredStore.Store(nil) + } + rs.deferredTasks.Unlock() +} + +func (rs *requestedStore) promoteDeferredTask() { + rs.deferredTasks.Lock() + if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { + rs.deferredTasks.Unlock() + return + } + task := rs.deferredTasks.tasks[0] + copy(rs.deferredTasks.tasks, rs.deferredTasks.tasks[1:]) + rs.deferredTasks.tasks[len(rs.deferredTasks.tasks)-1] = nil + rs.deferredTasks.tasks = rs.deferredTasks.tasks[:len(rs.deferredTasks.tasks)-1] + rs.deferredTasks.promoted = task + task.deferredStore.Store(rs) + rs.deferredTasks.Unlock() + + rs.scheduler.queue.Push(task) +} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 72cf1d0e93..967680d0fd 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -112,7 +112,7 @@ func newRegionRequestWorker( client: client, store: store, requestCache: newRequestCache(requestCacheSize, func() { - store.promoteDeferredTask(client.regionTaskQueue) + store.promoteDeferredTask() }), controlQueue: newControlQueue(), } @@ -389,10 +389,6 @@ func (s *regionRequestWorker) processRegionSendTask( ctx context.Context, conn *ConnAndClient, ) error { - if s.controlQueue == nil { - s.controlQueue = newControlQueue() - } - doSend := func(req *cdcpb.ChangeDataRequest) error { if err := conn.Client.Send(req); err != nil { log.Warn("region request worker send request to grpc stream failed", @@ -599,3 +595,7 @@ func (s *regionRequestWorker) clearPendingRegions() []regionInfo { regions = append(regions, s.requestCache.takeUnsentRegions()...) return regions } + +func (s *regionRequestWorker) releaseAdmittedRegionRequests() { + s.requestCache.clear() +} diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 2d329540d2..6ba37d064a 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -334,6 +334,7 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, } @@ -383,6 +384,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10), + controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, client: &subscriptionClient{}, } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index f100612d41..9380320429 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" - "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" @@ -34,7 +33,6 @@ import ( "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/utils/dynstream" - "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/prometheus/client_golang/prometheus" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/oracle" @@ -192,8 +190,6 @@ type subscriptionClient struct { pdClock pdutil.Clock lockResolver txnutil.LockResolver - stores sync.Map - ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] // the following three fields are used to manage feedback from ds and notify other goroutines mu sync.Mutex @@ -210,11 +206,8 @@ type subscriptionClient struct { // rangeTaskCh is used to receive range tasks. // The tasks will be handled in `handleRangeTask` goroutine. - rangeTaskCh chan rangeTask - // regionTaskQueue is used to receive region tasks with priority. - // The region will be handled in `handleRegions` goroutine. - regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask] - regionTaskSeq atomic.Uint64 + rangeTaskCh chan rangeTask + regionScheduler *regionRequestScheduler // resolveLockTaskCh is used to receive resolve lock tasks. // The tasks will be handled in `handleResolveLockTasks` goroutine. resolveLockTaskCh chan resolveLockTask @@ -234,7 +227,6 @@ func NewSubscriptionClient( subClient := &subscriptionClient{ config: config, - stores: sync.Map{}, pd: pd, regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), @@ -243,11 +235,11 @@ func NewSubscriptionClient( credential: credential, rangeTaskCh: make(chan rangeTask, 1024), - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), errCache: newErrCache(), } + subClient.regionScheduler = newRegionRequestScheduler(subClient) subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) subClient.totalSpans.spanMap = make(map[SubscriptionID]*subscribedSpan) @@ -319,16 +311,8 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { ).Set(float64(areaMetric.MemoryUsage())) } - pendingRegionReqCount := 0 - s.stores.Range(func(_, value any) bool { - store := value.(*requestedStore) - for _, worker := range store.requestWorkers { - pendingRegionReqCount += worker.requestCache.getPendingCount() - } - return true - }) - - metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending").Set(float64(pendingRegionReqCount)) + metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending"). + Set(float64(s.regionScheduler.pendingRequestCount())) count := 0 s.totalSpans.RLock() @@ -459,7 +443,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { g.Go(func() error { return s.updateMetrics(ctx) }) g.Go(func() error { return s.handleDSFeedBack(ctx) }) g.Go(func() error { return s.handleRangeTasks(ctx) }) - g.Go(func() error { return s.handleRegions(ctx, g) }) + g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) g.Go(func() error { return s.handleErrors(ctx) }) g.Go(func() error { return s.runResolveLockChecker(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) @@ -479,7 +463,9 @@ func (s *subscriptionClient) Close(ctx context.Context) error { s.cond.Broadcast() s.mu.Unlock() s.ds.Close() - s.regionTaskQueue.Close() + if s.regionScheduler != nil { + s.regionScheduler.close() + } return nil } @@ -491,7 +477,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Then send a special singleRegionInfo to regionRouter to deregister the table // from all TiKV instances. if rt.stopped.CompareAndSwap(false, true) { - s.enqueueDeregisterToAllStores(rt.subID, rt.filterLoop) + s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -525,245 +511,6 @@ func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { s.errCache.add(errInfo) } -// requestedStore represents a store that has been connected. -type requestedStore struct { - storeAddr string - // nextWorker is the round‑robin cursor used to select the next worker to attempt. - nextWorker atomic.Uint32 - // requestWorkers are fully created before requestedStore is published - // and remain immutable afterwards. - requestWorkers []*regionRequestWorker - - deferredTasks struct { - sync.Mutex - tasks []*regionPriorityTask - promoted *regionPriorityTask - } -} - -func (rs *requestedStore) addRegion( - ctx context.Context, region regionInfo, force bool, -) (bool, *regionRequestWorker, error) { - workers := rs.requestWorkers - - start := int(rs.nextWorker.Add(1)) % len(workers) - for i := range len(workers) { - worker := workers[(start+i)%len(workers)] - ok, err := worker.add(ctx, region, force) - if err != nil || ok { - return ok, worker, err - } - } - return false, nil, nil -} - -func (rs *requestedStore) canAdmitTask(task *regionPriorityTask) bool { - rs.deferredTasks.Lock() - defer rs.deferredTasks.Unlock() - - if rs.deferredTasks.promoted == task { - return true - } - return rs.deferredTasks.promoted == nil && len(rs.deferredTasks.tasks) == 0 -} - -func (rs *requestedStore) deferTask(task *regionPriorityTask) { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted == task { - rs.deferredTasks.promoted = nil - task.deferredStore.Store(nil) - rs.deferredTasks.tasks = append([]*regionPriorityTask{task}, rs.deferredTasks.tasks...) - } else { - rs.deferredTasks.tasks = append(rs.deferredTasks.tasks, task) - } - rs.deferredTasks.Unlock() -} - -func (rs *requestedStore) maybePromoteDeferredTask( - regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask], -) { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { - rs.deferredTasks.Unlock() - return - } - force := rs.deferredTasks.tasks[0].Priority() <= forcedPriorityBase - rs.deferredTasks.Unlock() - - if rs.hasRequestCapacity(force) { - rs.promoteDeferredTask(regionTaskQueue) - } -} - -func (rs *requestedStore) hasRequestCapacity(force bool) bool { - for _, worker := range rs.requestWorkers { - if worker.requestCache.canAdd(force) { - return true - } - } - return false -} - -func (rs *requestedStore) finishPromotedTask(task *regionPriorityTask) { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted == task { - rs.deferredTasks.promoted = nil - task.deferredStore.Store(nil) - } - rs.deferredTasks.Unlock() -} - -func (rs *requestedStore) promoteDeferredTask(regionTaskQueue *priorityqueue.PriorityQueue[*regionPriorityTask]) { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { - rs.deferredTasks.Unlock() - return - } - task := rs.deferredTasks.tasks[0] - copy(rs.deferredTasks.tasks, rs.deferredTasks.tasks[1:]) - rs.deferredTasks.tasks[len(rs.deferredTasks.tasks)-1] = nil - rs.deferredTasks.tasks = rs.deferredTasks.tasks[:len(rs.deferredTasks.tasks)-1] - rs.deferredTasks.promoted = task - task.deferredStore.Store(rs) - rs.deferredTasks.Unlock() - - regionTaskQueue.Push(task) -} - -func (s *subscriptionClient) newRegionPriorityTask(taskType TaskType, regionInfo regionInfo) *regionPriorityTask { - return newRegionPriorityTask(taskType, regionInfo, s.pdClock.CurrentTS(), s.regionTaskSeq.Add(1)) -} - -// handleRegions receives regionInfo from regionTaskQueue and attach rpcCtx to them, -// then send them to corresponding requestedStore. -func (s *subscriptionClient) handleRegions(ctx context.Context, eg *errgroup.Group) error { - cfg := config.GetGlobalServerConfig() - pendingRegionRequestQueueSize := cfg.Debug.Puller.PendingRegionRequestQueueSize - // Store creation is serialized by the single handleRegions loop. - getStore := func(storeAddr string) *requestedStore { - var rs *requestedStore - if v, ok := s.stores.Load(storeAddr); ok { - rs = v.(*requestedStore) - return rs - } - - perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.config.RegionRequestWorkerPerStore) - if perWorkerQueueSize <= 0 { - log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", - zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), - zap.Uint("regionRequestWorkerPerStore", s.config.RegionRequestWorkerPerStore)) - perWorkerQueueSize = 1 - } - - rs = &requestedStore{ - storeAddr: storeAddr, - requestWorkers: make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore), - } - for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { - requestWorker := newRegionRequestWorker(ctx, s, s.credential, eg, rs, perWorkerQueueSize) - rs.requestWorkers = append(rs.requestWorkers, requestWorker) - } - s.stores.Store(storeAddr, rs) - return rs - } - - defer func() { - s.stores.Range(func(_, value any) bool { - rs := value.(*requestedStore) - - for _, w := range rs.requestWorkers { - w.requestCache.clear() - } - - return true - }) - }() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - // Use blocking Pop to wait for tasks - regionTask, err := s.regionTaskQueue.Pop(ctx) - if err != nil { - if errors.Is(err, priorityqueue.ErrClosed) { - return nil - } - return err - } - - region := regionTask.GetRegionInfo() - if region.isStopped() { - s.enqueueDeregisterToAllStores(region.subscribedSpan.subID, region.filterLoop) - continue - } - - promotedStore := regionTask.deferredStore.Load() - region, ok := s.attachRPCContextForRegion(ctx, region) - // If attachRPCContextForRegion fails, the region will be re-scheduled. - if !ok { - if promotedStore != nil { - promotedStore.finishPromotedTask(regionTask) - promotedStore.promoteDeferredTask(s.regionTaskQueue) - } - continue - } - - store := getStore(region.rpcCtx.Addr) - if promotedStore != nil && promotedStore != store { - promotedStore.finishPromotedTask(regionTask) - promotedStore.promoteDeferredTask(s.regionTaskQueue) - } - if !store.canAdmitTask(regionTask) { - store.deferTask(regionTask) - store.maybePromoteDeferredTask(s.regionTaskQueue) - continue - } - force := regionTask.Priority() <= forcedPriorityBase - - var worker *regionRequestWorker - ok, worker, err = store.addRegion(ctx, region, force) - if err != nil { - log.Warn("subscription client add region request failed", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.Error(err)) - return err - } - - if !ok { - store.deferTask(regionTask) - store.maybePromoteDeferredTask(s.regionTaskQueue) - continue - } - store.finishPromotedTask(regionTask) - - log.Debug("subscription client will request a region", - zap.Uint64("workID", worker.workerID), - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", store.storeAddr)) - } -} - -func (s *subscriptionClient) enqueueDeregisterToAllStores(subID SubscriptionID, filterLoop bool) { - s.stores.Range(func(_ any, value any) bool { - rs := value.(*requestedStore) - for _, worker := range rs.requestWorkers { - if worker.controlQueue == nil { - worker.controlQueue = newControlQueue() - } - worker.controlQueue.push(deregisterRequest{ - subID: subID, - filterLoop: filterLoop, - }) - } - return true - }) -} - func (s *subscriptionClient) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.regionCache.GetTiKVRPCContext(bo, region.verID, kvclientv2.ReplicaReadLeader, 0) @@ -883,8 +630,8 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( } } -// scheduleRegionRequest locks the region's range and send the region to regionTaskQueue, -// which will be handled by handleRegions. +// scheduleRegionRequest locks the region's range and submits it to the region +// request scheduler. func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region regionInfo, priority TaskType) { lockRangeResult := region.subscribedSpan.rangeLock.LockRange( ctx, region.span.StartKey, region.span.EndKey, region.verID.GetID(), region.verID.GetVer()) @@ -896,7 +643,7 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState - s.regionTaskQueue.Push(s.newRegionPriorityTask(priority, region)) + s.regionScheduler.submit(priority, region) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, priority) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 0b56f05caa..b4d921d2aa 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -279,11 +279,11 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() client.pdClock = pdutil.NewClock4Test() + client.regionScheduler = newRegionRequestScheduler(client) rawSpan := heartbeatpb.TableSpan{ TableID: 1, @@ -298,9 +298,9 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) worker := ®ionRequestWorker{controlQueue: newControlQueue()} - store := &requestedStore{storeAddr: "store-1"} + store := &requestedStore{scheduler: client.regionScheduler, storeAddr: "store-1"} store.requestWorkers = []*regionRequestWorker{worker} - client.stores.Store(store.storeAddr, store) + client.regionScheduler.stores.Store(store.storeAddr, store) client.setTableStopped(span) @@ -386,11 +386,11 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { client := &subscriptionClient{ - ds: &mockDynamicStream{}, - regionTaskQueue: priorityqueue.New[*regionPriorityTask](), + ds: &mockDynamicStream{}, } client.ctx, client.cancel = context.WithCancel(context.Background()) client.cond = sync.NewCond(&client.mu) + client.regionScheduler = newRegionRequestScheduler(client) client.paused.Store(true) @@ -418,14 +418,15 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { ctx := context.Background() client := &subscriptionClient{} + scheduler := newRegionRequestScheduler(client) worker := ®ionRequestWorker{ requestCache: newRequestCache(1), controlQueue: newControlQueue(), } - store := &requestedStore{storeAddr: "store-1"} + store := &requestedStore{scheduler: scheduler, storeAddr: "store-1"} store.requestWorkers = []*regionRequestWorker{worker} - client.stores.Store(store.storeAddr, store) + scheduler.stores.Store(store.storeAddr, store) dummyRegion := regionInfo{ subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, @@ -435,7 +436,7 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { require.NoError(t, err) require.True(t, ok) - client.enqueueDeregisterToAllStores(SubscriptionID(1), true) + scheduler.broadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.ch.Len()) req, ok := worker.controlQueue.tryPop() require.True(t, ok) @@ -446,7 +447,8 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { func TestRequestedStoreDeferredTasksBlockSameStore(t *testing.T) { queue := priorityqueue.New[*regionPriorityTask]() - store := &requestedStore{storeAddr: "store-1"} + scheduler := ®ionRequestScheduler{queue: queue} + store := &requestedStore{scheduler: scheduler, storeAddr: "store-1"} currentTs := oracle.GoTimeToTS(time.Now()) span := &subscribedSpan{subID: SubscriptionID(1)} span.resolvedTs.Store(currentTs) @@ -456,23 +458,23 @@ func TestRequestedStoreDeferredTasksBlockSameStore(t *testing.T) { secondTask := newRegionPriorityTask(TaskHighPrior, region, currentTs, 2) store.deferTask(firstTask) - require.False(t, store.canAdmitTask(secondTask)) + require.True(t, store.hasDeferredTaskAhead(secondTask)) - store.promoteDeferredTask(queue) + store.promoteDeferredTask() promoted, err := queue.Pop(t.Context()) require.NoError(t, err) require.Same(t, firstTask, promoted) - require.True(t, store.canAdmitTask(firstTask)) - require.False(t, store.canAdmitTask(secondTask)) + require.False(t, store.hasDeferredTaskAhead(firstTask)) + require.True(t, store.hasDeferredTaskAhead(secondTask)) store.deferTask(firstTask) - store.promoteDeferredTask(queue) + store.promoteDeferredTask() promoted, err = queue.Pop(t.Context()) require.NoError(t, err) require.Same(t, firstTask, promoted) store.finishPromotedTask(firstTask) - require.True(t, store.canAdmitTask(secondTask)) + require.False(t, store.hasDeferredTaskAhead(secondTask)) } func TestSubscriptionWithFailedTiKV(t *testing.T) { From 9fedd36d83fe771929d026e0596561f2235f34fc Mon Sep 17 00:00:00 2001 From: lidezhu Date: Thu, 25 Jun 2026 22:37:46 -0700 Subject: [PATCH 12/21] refactor --- .../logpuller/region_request_scheduler.go | 60 ++++++++++--- logservice/logpuller/region_request_worker.go | 42 +++++---- .../logpuller/region_request_worker_test.go | 31 ++++--- logservice/logpuller/subscription_client.go | 89 +++++++------------ .../logpuller/subscription_client_test.go | 6 +- 5 files changed, 121 insertions(+), 107 deletions(-) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 2c8e739392..9714938c20 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -21,7 +21,10 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/config" "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/utils/priorityqueue" + kvclientv2 "github.com/tikv/client-go/v2/kv" + "github.com/tikv/client-go/v2/tikv" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) @@ -29,7 +32,11 @@ import ( // regionRequestScheduler owns region request admission from the global // priority queue to per-store request workers. type regionRequestScheduler struct { - client *subscriptionClient + config *SubscriptionClientConfig + upstream *upstreamHandle + + pushRegionEventToDS func(SubscriptionID, regionEvent) + onRegionFail func(regionErrorInfo) queue *priorityqueue.PriorityQueue[*regionPriorityTask] seq atomic.Uint64 @@ -39,13 +46,16 @@ type regionRequestScheduler struct { func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { return ®ionRequestScheduler{ - client: client, - queue: priorityqueue.New[*regionPriorityTask](), + config: client.config, + upstream: client.upstream, + pushRegionEventToDS: client.pushRegionEventToDS, + onRegionFail: client.onRegionFail, + queue: priorityqueue.New[*regionPriorityTask](), } } func (s *regionRequestScheduler) submit(taskType TaskType, region regionInfo) { - task := newRegionPriorityTask(taskType, region, s.client.pdClock.CurrentTS(), s.seq.Add(1)) + task := newRegionPriorityTask(taskType, region, s.upstream.pdClock.CurrentTS(), s.seq.Add(1)) s.queue.Push(task) } @@ -54,7 +64,7 @@ func (s *regionRequestScheduler) close() { s.releaseAdmittedRegionRequests() } -func (s *regionRequestScheduler) pendingRequestCount() int { +func (s *regionRequestScheduler) updateMetrics() { count := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) @@ -63,7 +73,8 @@ func (s *regionRequestScheduler) pendingRequestCount() int { } return true }) - return count + metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending"). + Set(float64(count)) } func (s *regionRequestScheduler) broadcastDeregister(subID SubscriptionID, filterLoop bool) { @@ -79,6 +90,23 @@ func (s *regionRequestScheduler) broadcastDeregister(subID SubscriptionID, filte }) } +func (s *regionRequestScheduler) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext(bo, region.verID, kvclientv2.ReplicaReadLeader, 0) + if rpcCtx != nil { + region.rpcCtx = rpcCtx + return region, true + } + if err != nil { + log.Debug("subscription client get rpc context fail", + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.Error(err)) + } + s.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + return region, false +} + // releaseAdmittedRegionRequests releases region requests admitted to store // workers during scheduler shutdown. func (s *regionRequestScheduler) releaseAdmittedRegionRequests() { @@ -102,21 +130,29 @@ func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) er return rs } - perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.client.config.RegionRequestWorkerPerStore) + perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.config.RegionRequestWorkerPerStore) if perWorkerQueueSize <= 0 { log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), - zap.Uint("regionRequestWorkerPerStore", s.client.config.RegionRequestWorkerPerStore)) + zap.Uint("regionRequestWorkerPerStore", s.config.RegionRequestWorkerPerStore)) perWorkerQueueSize = 1 } rs = &requestedStore{ scheduler: s, storeAddr: storeAddr, - requestWorkers: make([]*regionRequestWorker, 0, s.client.config.RegionRequestWorkerPerStore), + requestWorkers: make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore), } - for i := uint(0); i < s.client.config.RegionRequestWorkerPerStore; i++ { - requestWorker := newRegionRequestWorker(ctx, s.client, s.client.credential, eg, rs, perWorkerQueueSize) + for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { + requestWorker := newRegionRequestWorker( + ctx, + eg, + rs, + perWorkerQueueSize, + s.upstream, + s.pushRegionEventToDS, + s.onRegionFail, + ) rs.requestWorkers = append(rs.requestWorkers, requestWorker) } s.stores.Store(storeAddr, rs) @@ -144,7 +180,7 @@ func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) er } promotedStore := regionTask.deferredStore.Load() - region, ok := s.client.attachRPCContextForRegion(ctx, region) + region, ok := s.attachRPCContextForRegion(ctx, region) if !ok { if promotedStore != nil { promotedStore.finishPromotedTask(regionTask) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 967680d0fd..c72c77dfe1 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/kvproto/pkg/kvrpcpb" "github.com/pingcap/log" cerror "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/pkg/version" @@ -79,10 +80,12 @@ func (q *controlQueue) tryPop() (deregisterRequest, bool) { type regionRequestWorker struct { workerID uint64 - client *subscriptionClient - store *requestedStore + upstream *upstreamHandle + pushRegionEventToDS func(SubscriptionID, regionEvent) + onRegionFail func(regionErrorInfo) + // we must always get a region to request before create a grpc stream. // only in this way we can avoid to try to connect to an offline store infinitely. preFetchForConnecting *regionReq @@ -101,16 +104,19 @@ type regionRequestWorker struct { func newRegionRequestWorker( ctx context.Context, - client *subscriptionClient, - credential *security.Credential, g *errgroup.Group, store *requestedStore, requestCacheSize int, + upstream *upstreamHandle, + pushRegionEventToDS func(SubscriptionID, regionEvent), + onRegionFail func(regionErrorInfo), ) *regionRequestWorker { worker := ®ionRequestWorker{ - workerID: workerIDGen.Add(1), - client: client, - store: store, + workerID: workerIDGen.Add(1), + store: store, + upstream: upstream, + pushRegionEventToDS: pushRegionEventToDS, + onRegionFail: onRegionFail, requestCache: newRequestCache(requestCacheSize, func() { store.promoteDeferredTask() }), @@ -144,7 +150,7 @@ func newRegionRequestWorker( return err } var regionErr error - if err := version.CheckStoreVersion(ctx, worker.client.pd); err != nil { + if err := version.CheckStoreVersion(ctx, worker.upstream.pd); err != nil { if errors.Cause(err) == context.Canceled { return nil } @@ -158,7 +164,7 @@ func newRegionRequestWorker( regionErr = &storeStreamErr{} } } else { - if canceled := worker.run(ctx, credential); canceled { + if canceled := worker.run(ctx, worker.upstream.credential); canceled { return nil } regionErr = &storeStreamErr{} @@ -169,7 +175,7 @@ func newRegionRequestWorker( regionEvent := regionEvent{ states: []*regionFeedState{state}, } - worker.client.pushRegionEventToDS(subID, regionEvent) + worker.pushRegionEventToDS(subID, regionEvent) } } // The store may fail forever, so we need try to re-schedule all pending regions. @@ -178,7 +184,7 @@ func newRegionRequestWorker( // It means it's a special task for stopping the table. continue } - client.onRegionFail(newRegionErrorInfo(region, regionErr)) + worker.onRegionFail(newRegionErrorInfo(region, regionErr)) } if err := util.Hang(ctx, time.Second); err != nil { return err @@ -311,7 +317,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) default: log.Panic("unknown event type", zap.Any("event", event)) } - s.client.pushRegionEventToDS(subscriptionID, regionEvent) + s.pushRegionEventToDS(subscriptionID, regionEvent) } else { switch event.Event.(type) { case *cdcpb.Event_Error: @@ -333,7 +339,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.ResolvedTs) { subscriptionID := SubscriptionID(resolvedTsEvent.RequestId) metricsResolvedTsCount.Add(float64(len(resolvedTsEvent.Regions))) - s.client.metrics.batchResolvedSize.Observe(float64(len(resolvedTsEvent.Regions))) + metrics.BatchResolvedEventSize.WithLabelValues("event-store").Observe(float64(len(resolvedTsEvent.Regions))) // TODO: resolvedTsEvent.Ts be 0 is impossible, we need find the root cause. if resolvedTsEvent.Ts == 0 { log.Warn("region request worker receives a resolved ts event with zero value, ignore it", @@ -354,7 +360,7 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res if len(resolvedStates) == 0 { return } - s.client.pushRegionEventToDS(subscriptionID, regionEvent{ + s.pushRegionEventToDS(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: resolvedStates, }) @@ -404,7 +410,7 @@ func (s *regionRequestWorker) processRegionSendTask( } sendDeregister := func(req deregisterRequest) error { changeDataReq := &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: s.upstream.clusterID, TicdcVersion: version.ReleaseSemver()}, RequestId: uint64(req.subID), Request: &cdcpb.ChangeDataRequest_Deregister_{ Deregister: &cdcpb.ChangeDataRequest_Deregister{}, @@ -419,7 +425,7 @@ func (s *regionRequestWorker) processRegionSendTask( regionEvent := regionEvent{ states: []*regionFeedState{state}, } - s.client.pushRegionEventToDS(req.subID, regionEvent) + s.pushRegionEventToDS(req.subID, regionEvent) } return nil } @@ -472,7 +478,7 @@ func (s *regionRequestWorker) processRegionSendTask( // It can be skipped directly because there must be no pending states from // the stopped subscribedTable, or the special singleRegionInfo for stopping // the table will be handled later. - s.client.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) + s.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) regionReq.finish() } else { state := newRegionFeedState(region, uint64(subID), s, regionReq) @@ -503,7 +509,7 @@ func (s *regionRequestWorker) processRegionSendTask( func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.ChangeDataRequest { return &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.client.clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: s.upstream.clusterID, TicdcVersion: version.ReleaseSemver()}, RegionId: region.verID.GetID(), RequestId: uint64(region.subscribedSpan.subID), RegionEpoch: region.rpcCtx.Meta.RegionEpoch, diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 6ba37d064a..54c28565c2 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -23,7 +23,6 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/ticdc/logservice/logpuller/regionlock" "github.com/pingcap/ticdc/utils/dynstream" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "github.com/tikv/client-go/v2/tikv" "google.golang.org/grpc" @@ -182,13 +181,9 @@ func (m *mockRegionEventDynamicStream) GetMetrics() dynstream.Metrics[int, Subsc func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mockRegionEventDynamicStream, *cdcpb.ResolvedTs) { ds := &mockRegionEventDynamicStream{} + client := &subscriptionClient{ds: ds} worker := ®ionRequestWorker{ - client: &subscriptionClient{ - metrics: sharedClientMetrics{ - batchResolvedSize: prometheus.ObserverFunc(func(float64) {}), - }, - ds: ds, - }, + pushRegionEventToDS: client.pushRegionEventToDS, } worker.requestedRegions.subscriptions = map[SubscriptionID]regionFeedStates{ 1: make(regionFeedStates, regionCount), @@ -218,7 +213,7 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT return } states := resolvedStates - s.client.pushRegionEventToDS(subscriptionID, regionEvent{ + s.pushRegionEventToDS(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: states, }) @@ -333,10 +328,12 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, + requestCache: newRequestCache(10), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + upstream: &upstreamHandle{}, + pushRegionEventToDS: func(SubscriptionID, regionEvent) {}, + onRegionFail: func(regionErrorInfo) {}, } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) @@ -383,10 +380,12 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, - client: &subscriptionClient{}, + requestCache: newRequestCache(10), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + upstream: &upstreamHandle{}, + pushRegionEventToDS: func(SubscriptionID, regionEvent) {}, + onRegionFail: func(regionErrorInfo) {}, } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 9380320429..a0e8d4ed5a 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -33,8 +33,6 @@ import ( "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/utils/dynstream" - "github.com/prometheus/client_golang/prometheus" - kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" @@ -154,8 +152,12 @@ type SubscriptionClientConfig struct { RegionRequestWorkerPerStore uint } -type sharedClientMetrics struct { - batchResolvedSize prometheus.Observer +type upstreamHandle struct { + pd pd.Client + credential *security.Credential + pdClock pdutil.Clock + regionCache *tikv.RegionCache + clusterID uint64 } // subscriptionClient is used to subscribe events of table ranges from TiKV. @@ -179,15 +181,11 @@ type SubscriptionClient interface { } type subscriptionClient struct { - ctx context.Context - cancel context.CancelFunc - config *SubscriptionClientConfig - metrics sharedClientMetrics - clusterID uint64 - - pd pd.Client - regionCache *tikv.RegionCache - pdClock pdutil.Clock + ctx context.Context + cancel context.CancelFunc + config *SubscriptionClientConfig + upstream *upstreamHandle + lockResolver txnutil.LockResolver ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] @@ -196,9 +194,6 @@ type subscriptionClient struct { cond *sync.Cond paused atomic.Bool - // the credential to connect tikv - credential *security.Credential - totalSpans struct { sync.RWMutex spanMap map[SubscriptionID]*subscribedSpan @@ -224,22 +219,23 @@ func NewSubscriptionClient( lockResolver txnutil.LockResolver, credential *security.Credential, ) SubscriptionClient { + regionCache := appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache) + pdClock := appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock) subClient := &subscriptionClient{ config: config, - - pd: pd, - regionCache: appcontext.GetService[*tikv.RegionCache](appcontext.RegionCache), - pdClock: appcontext.GetService[pdutil.Clock](appcontext.DefaultPDClock), + upstream: &upstreamHandle{ + pd: pd, + credential: credential, + pdClock: pdClock, + regionCache: regionCache, + }, lockResolver: lockResolver, - credential: credential, - rangeTaskCh: make(chan rangeTask, 1024), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), errCache: newErrCache(), } - subClient.regionScheduler = newRegionRequestScheduler(subClient) subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) subClient.totalSpans.spanMap = make(map[SubscriptionID]*subscribedSpan) @@ -259,7 +255,7 @@ func NewSubscriptionClient( subClient.ds = ds subClient.cond = sync.NewCond(&subClient.mu) - subClient.initMetrics() + subClient.regionScheduler = newRegionRequestScheduler(subClient) return subClient } @@ -272,11 +268,6 @@ func (s *subscriptionClient) AllocSubscriptionID() SubscriptionID { return SubscriptionID(subscriptionIDGen.Add(1)) } -func (s *subscriptionClient) initMetrics() { - // TODO: fix metrics - s.metrics.batchResolvedSize = metrics.BatchResolvedEventSize.WithLabelValues("event-store") -} - func (s *subscriptionClient) updateMetrics(ctx context.Context) error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -311,8 +302,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { ).Set(float64(areaMetric.MemoryUsage())) } - metrics.SubscriptionClientRequestedRegionCount.WithLabelValues("pending"). - Set(float64(s.regionScheduler.pendingRequestCount())) + s.regionScheduler.updateMetrics() count := 0 s.totalSpans.RLock() @@ -432,11 +422,11 @@ func (s *subscriptionClient) handleDSFeedBack(ctx context.Context) error { func (s *subscriptionClient) Run(ctx context.Context) error { // s.consume = consume - if s.pd == nil { + if s.upstream == nil || s.upstream.pd == nil { log.Warn("subscription client should be in test mode, skip run") return nil } - s.clusterID = s.pd.GetClusterID(ctx) + s.upstream.clusterID = s.upstream.pd.GetClusterID(ctx) g, ctx := errgroup.WithContext(ctx) @@ -511,23 +501,6 @@ func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { s.errCache.add(errInfo) } -func (s *subscriptionClient) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - rpcCtx, err := s.regionCache.GetTiKVRPCContext(bo, region.verID, kvclientv2.ReplicaReadLeader, 0) - if rpcCtx != nil { - region.rpcCtx = rpcCtx - return region, true - } - if err != nil { - log.Debug("subscription client get rpc context fail", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.Error(err)) - } - s.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) - return region, false -} - func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -572,7 +545,7 @@ func (s *subscriptionClient) divideSpanAndScheduleRegionRequests( zap.Any("span", common.FormatTableSpan(&nextSpan))) backoff := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - regions, err := s.regionCache.BatchLoadRegionsWithKeyRange(backoff, nextSpan.StartKey, nextSpan.EndKey, limit) + regions, err := s.upstream.regionCache.BatchLoadRegionsWithKeyRange(backoff, nextSpan.StartKey, nextSpan.EndKey, limit) if err != nil { log.Warn("subscription client load regions failed", zap.Uint64("subscriptionID", uint64(subscribedSpan.subID)), @@ -694,7 +667,7 @@ func (s *subscriptionClient) doHandleError(ctx context.Context, errInfo regionEr innerErr := eerr.err if notLeader := innerErr.GetNotLeader(); notLeader != nil { metricFeedNotLeaderCounter.Inc() - s.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) + s.upstream.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) return nil } @@ -744,13 +717,13 @@ func (s *subscriptionClient) doHandleError(ctx context.Context, errInfo regionEr metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) // cannot get the store the region belongs to, so we need to reload the region. - s.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) + s.upstream.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) s.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) return nil case *storeStreamErr: metricStoreSendRequestErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - s.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) + s.upstream.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) return nil case *requestCancelledErr: @@ -796,13 +769,13 @@ func (s *subscriptionClient) runResolveLockChecker(ctx context.Context) error { case <-resolveLockTicker.C: } - physical, logic, err := s.pd.GetTS(ctx) + physical, logic, err := s.upstream.pd.GetTS(ctx) if err != nil { log.Warn("get ts from pd failed", zap.Error(err)) continue } currentTs := oracle.ComposeTS(physical, logic) - currentTime := s.pdClock.CurrentTime() + currentTime := s.upstream.pdClock.CurrentTime() s.totalSpans.Lock() for _, subSpan := range s.totalSpans.spanMap { if subSpan != nil { @@ -874,7 +847,7 @@ func (s *subscriptionClient) logSlowRegions(ctx context.Context) error { case <-ticker.C: } - currTime := s.pdClock.CurrentTime() + currTime := s.upstream.pdClock.CurrentTime() s.totalSpans.RLock() slowInitializeRegion := 0 for subscriptionID, rt := range s.totalSpans.spanMap { @@ -972,7 +945,7 @@ func (s *subscriptionClient) GetResolvedTsLag() float64 { if pullerMinResolvedTs == 0 { return 0 } - pdTime := s.pdClock.CurrentTime() + pdTime := s.upstream.pdClock.CurrentTime() phyResolvedTs := oracle.ExtractPhysical(pullerMinResolvedTs) lag := float64(oracle.GetPhysical(pdTime)-phyResolvedTs) / 1e3 return lag diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index b4d921d2aa..1ad6dbd05f 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -57,6 +57,7 @@ func TestGenerateResolveLockTask(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 10), resolveLockRateLimiter: newResolveLockRateLimiter(), + upstream: &upstreamHandle{pdClock: pdutil.NewClock4Test()}, } client.ctx, client.cancel = context.WithCancel(context.Background()) rawSpan := heartbeatpb.TableSpan{ @@ -69,7 +70,6 @@ func TestGenerateResolveLockTask(t *testing.T) { span := client.newSubscribedSpan(SubscriptionID(1), rawSpan, 100, consumeKVEvents, advanceResolvedTs, 0, false) client.totalSpans.spanMap = make(map[SubscriptionID]*subscribedSpan) client.totalSpans.spanMap[SubscriptionID(1)] = span - client.pdClock = pdutil.NewClock4Test() // Lock a range, and then ResolveLock will trigger a task for it. res := span.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) @@ -279,10 +279,10 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), + upstream: &upstreamHandle{pdClock: pdutil.NewClock4Test()}, } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() - client.pdClock = pdutil.NewClock4Test() client.regionScheduler = newRegionRequestScheduler(client) rawSpan := heartbeatpb.TableSpan{ @@ -417,7 +417,7 @@ func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { ctx := context.Background() - client := &subscriptionClient{} + client := &subscriptionClient{upstream: &upstreamHandle{}} scheduler := newRegionRequestScheduler(client) worker := ®ionRequestWorker{ From 3bdd89462207ed9eff156ec98e22f7c89003db65 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 26 Jun 2026 13:31:18 -0700 Subject: [PATCH 13/21] more refactor --- logservice/logpuller/region_event_handler.go | 7 +- logservice/logpuller/region_event_sink.go | 133 ++++++++ .../logpuller/region_failure_reporter.go | 300 ++++++++++++++++ .../logpuller/region_request_scheduler.go | 20 +- logservice/logpuller/region_request_worker.go | 32 +- .../logpuller/region_request_worker_test.go | 29 +- logservice/logpuller/subscription_client.go | 323 +----------------- .../logpuller/subscription_client_test.go | 27 +- 8 files changed, 506 insertions(+), 365 deletions(-) create mode 100644 logservice/logpuller/region_event_sink.go create mode 100644 logservice/logpuller/region_failure_reporter.go diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 7e96008c77..272bed7d9a 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -87,7 +87,8 @@ func (event regionEvent) mustFirstState() *regionFeedState { } type regionEventHandler struct { - subClient *subscriptionClient + eventSink *regionEventSink + failureReporter *regionFailureReporter } func (h *regionEventHandler) Path(event regionEvent) SubscriptionID { @@ -159,7 +160,7 @@ func (h *regionEventHandler) Handle(span *subscribedSpan, events ...regionEvent) metricConsumeKVEventsCallbackDurationAdvanceResolvedTs.Observe(time.Since(start).Seconds()) start = time.Now() - h.subClient.wakeSubscription(span.subID) + h.eventSink.Wake(span.subID) metricConsumeKVEventsCallbackDurationWakeSubscription.Observe(time.Since(start).Seconds()) }) // if not await, the wake callback will not be called, we need clear the cache manually. @@ -255,7 +256,7 @@ func (h *regionEventHandler) handleRegionError(state *regionFeedState) { } if stepsToRemoved { worker.takeRegionState(SubscriptionID(state.requestID), state.getRegionID()) - h.subClient.onRegionFail(newRegionErrorInfo(state.getRegionInfo(), err)) + h.failureReporter.Report(newRegionErrorInfo(state.getRegionInfo(), err)) } } diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go new file mode 100644 index 0000000000..34a27dd177 --- /dev/null +++ b/logservice/logpuller/region_event_sink.go @@ -0,0 +1,133 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/utils/dynstream" + "go.uber.org/zap" +) + +type regionEventSink struct { + ctx context.Context + ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] + + mu sync.Mutex + cond *sync.Cond + paused atomic.Bool +} + +func newRegionEventSink(ctx context.Context, failureReporter *regionFailureReporter) *regionEventSink { + sink := ®ionEventSink{ctx: ctx} + sink.cond = sync.NewCond(&sink.mu) + + option := dynstream.NewOption() + // Note: it is max batch size of the kv sent from tikv(not committed rows) + option.BatchCount = 1024 + // TODO: Set `UseBuffer` to true until we refactor the `regionEventHandler.Handle` method so that it doesn't call any method of the dynamic stream. Currently, if `UseBuffer` is set to false, there will be a deadlock: + // ds.handleLoop fetch events from `ch` -> regionEventHandler.Handle -> ds.RemovePath -> send event to `ch` + option.UseBuffer = true + option.EnableMemoryControl = true + ds := dynstream.NewParallelDynamicStream( + "log-puller", + ®ionEventHandler{eventSink: sink, failureReporter: failureReporter}, + option, + ) + ds.Start() + sink.ds = ds + return sink +} + +func (s *regionEventSink) AddPath(rt *subscribedSpan) { + areaSetting := dynstream.NewAreaSettingsWithMaxPendingSize(1*1024*1024*1024, dynstream.MemoryControlForPuller, "logPuller") // 1GB + if err := s.ds.AddPath(rt.subID, rt, areaSetting); err != nil { + log.Warn("subscription client add path failed", + zap.Uint64("subscriptionID", uint64(rt.subID)), + zap.Error(err)) + } +} + +func (s *regionEventSink) RemovePath(subID SubscriptionID) error { + return s.ds.RemovePath(subID) +} + +func (s *regionEventSink) Wake(subID SubscriptionID) { + s.ds.Wake(subID) +} + +func (s *regionEventSink) Push(subID SubscriptionID, event regionEvent) { + // fast path + if !s.paused.Load() { + s.ds.Push(subID, event) + return + } + // slow path: wait until paused is false + s.mu.Lock() + for s.paused.Load() { + select { + case <-s.ctx.Done(): + s.mu.Unlock() + return + default: + s.cond.Wait() + } + } + s.mu.Unlock() + s.ds.Push(subID, event) +} + +func (s *regionEventSink) Run(ctx context.Context) error { + return s.handleFeedback(ctx) +} + +func (s *regionEventSink) handleFeedback(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + case feedback := <-s.ds.Feedback(): + switch feedback.FeedbackType { + case dynstream.PauseArea: + s.mu.Lock() + s.paused.Store(true) + s.mu.Unlock() + log.Info("subscription client pause push region event") + case dynstream.ResumeArea: + s.mu.Lock() + s.paused.Store(false) + s.cond.Broadcast() + s.mu.Unlock() + log.Info("subscription client resume push region event") + case dynstream.ReleasePath, dynstream.ResumePath: + // Ignore it, because it is no need to pause and resume a path in puller. + } + } + } +} + +func (s *regionEventSink) Metrics() dynstream.Metrics[int, SubscriptionID] { + return s.ds.GetMetrics() +} + +func (s *regionEventSink) Close() { + s.mu.Lock() + s.paused.Store(false) + s.cond.Broadcast() + s.mu.Unlock() + s.ds.Close() +} diff --git a/logservice/logpuller/region_failure_reporter.go b/logservice/logpuller/region_failure_reporter.go new file mode 100644 index 0000000000..8ffe5271ab --- /dev/null +++ b/logservice/logpuller/region_failure_reporter.go @@ -0,0 +1,300 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/tikv/client-go/v2/tikv" + "go.uber.org/zap" +) + +const ( + // TiCDC always interacts with region leader, every time something goes wrong, + // failed region will be reloaded via `BatchLoadRegionsWithKeyRange` API. So we + // don't need to force reload region anymore. + regionScheduleReload = false +) + +var ( + metricFeedNotLeaderCounter = metrics.EventFeedErrorCounter.WithLabelValues("NotLeader") + metricFeedEpochNotMatchCounter = metrics.EventFeedErrorCounter.WithLabelValues("EpochNotMatch") + metricFeedRegionNotFoundCounter = metrics.EventFeedErrorCounter.WithLabelValues("RegionNotFound") + metricFeedDuplicateRequestCounter = metrics.EventFeedErrorCounter.WithLabelValues("DuplicateRequest") + metricFeedUnknownErrorCounter = metrics.EventFeedErrorCounter.WithLabelValues("Unknown") + metricFeedRPCCtxUnavailable = metrics.EventFeedErrorCounter.WithLabelValues("RPCCtxUnavailable") + metricGetStoreErr = metrics.EventFeedErrorCounter.WithLabelValues("GetStoreErr") + metricStoreSendRequestErr = metrics.EventFeedErrorCounter.WithLabelValues("SendRequestToStore") + metricKvIsBusyCounter = metrics.EventFeedErrorCounter.WithLabelValues("KvIsBusy") + metricKvCongestedCounter = metrics.EventFeedErrorCounter.WithLabelValues("KvCongested") +) + +type regionFailureReporter struct { + cache *errCache + + upstream *upstreamHandle + onTableDrained func(*subscribedSpan) + scheduleRegionRequest func(context.Context, regionInfo, TaskType) + scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, TaskType) +} + +func newRegionFailureReporter( + upstream *upstreamHandle, + onTableDrained func(*subscribedSpan), + scheduleRegionRequest func(context.Context, regionInfo, TaskType), + scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, TaskType), +) *regionFailureReporter { + return ®ionFailureReporter{ + cache: newErrCache(), + upstream: upstream, + onTableDrained: onTableDrained, + scheduleRegionRequest: scheduleRegionRequest, + scheduleRangeRequest: scheduleRangeRequest, + } +} + +// Report admits a region failure into the recovery pipeline. It releases the +// corresponding range lock before enqueueing the failure so new range tasks are +// not blocked by stale region ownership. +func (r *regionFailureReporter) Report(errInfo regionErrorInfo) { + if errInfo.subscribedSpan.rangeLock.UnlockRange( + errInfo.span.StartKey, errInfo.span.EndKey, + errInfo.verID.GetID(), errInfo.verID.GetVer(), errInfo.resolvedTs()) { + r.onTableDrained(errInfo.subscribedSpan) + return + } + r.cache.add(errInfo) +} + +func (r *regionFailureReporter) Run(ctx context.Context) error { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + handlePendingFailures := func() error { + for { + batch := r.cache.popBatch(errCacheDispatchBatchSize) + for _, errInfo := range batch { + select { + case <-ctx.Done(): + log.Info("subscription client handle errors and exit") + return ctx.Err() + default: + } + if err := r.handleError(ctx, errInfo); err != nil { + return err + } + } + if len(batch) < errCacheDispatchBatchSize { + return nil + } + } + } + for { + select { + case <-ctx.Done(): + log.Info("subscription client handle errors and exit") + return ctx.Err() + case <-ticker.C: + if err := handlePendingFailures(); err != nil { + return err + } + case <-r.cache.notify: + if err := handlePendingFailures(); err != nil { + return err + } + } + } +} + +func (r *regionFailureReporter) handleError(ctx context.Context, errInfo regionErrorInfo) error { + err := errors.Cause(errInfo.err) + //nolint:errorlint // errors.Cause unwraps repository errors before the concrete type check. + if _, requestCancelled := err.(*requestCancelledErr); !requestCancelled { + log.Debug("cdc region error", + zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), + zap.Uint64("regionID", errInfo.verID.GetID()), + zap.Error(err)) + } + + //nolint:errorlint // converting large type switch to errors.As is a significant refactor + switch eerr := err.(type) { + case *eventError: + innerErr := eerr.err + if notLeader := innerErr.GetNotLeader(); notLeader != nil { + metricFeedNotLeaderCounter.Inc() + r.upstream.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) + r.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + return nil + } + if innerErr.GetEpochNotMatch() != nil { + metricFeedEpochNotMatchCounter.Inc() + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + return nil + } + if innerErr.GetRegionNotFound() != nil { + metricFeedRegionNotFoundCounter.Inc() + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + return nil + } + if innerErr.GetCongested() != nil { + metricKvCongestedCounter.Inc() + r.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskLowPrior) + return nil + } + if innerErr.GetServerIsBusy() != nil { + metricKvIsBusyCounter.Inc() + r.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskLowPrior) + return nil + } + if duplicated := innerErr.GetDuplicateRequest(); duplicated != nil { + // TODO(qupeng): It's better to add a new machanism to deregister one region. + metricFeedDuplicateRequestCounter.Inc() + return errors.New("duplicate request") + } + if compatibility := innerErr.GetCompatibility(); compatibility != nil { + return errors.ErrVersionIncompatible.GenWithStackByArgs(compatibility) + } + if mismatch := innerErr.GetClusterIdMismatch(); mismatch != nil { + return errors.ErrClusterIDMismatch.GenWithStackByArgs(mismatch.Current, mismatch.Request) + } + + log.Warn("empty or unknown cdc error", + zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), + zap.Stringer("error", innerErr)) + metricFeedUnknownErrorCounter.Inc() + r.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + return nil + case *rpcCtxUnavailableErr: + metricFeedRPCCtxUnavailable.Inc() + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + return nil + case *getStoreErr: + metricGetStoreErr.Inc() + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + // cannot get the store the region belongs to, so we need to reload the region. + r.upstream.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) + r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) + return nil + case *storeStreamErr: + metricStoreSendRequestErr.Inc() + bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) + r.upstream.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) + r.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + return nil + case *requestCancelledErr: + // the corresponding subscription has been unsubscribed, just ignore. + return nil + default: + // TODO(qupeng): for some errors it's better to just deregister the region from TiKVs. + log.Warn("subscription client meets an internal error, fail the changefeed", + zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), + zap.Error(err)) + return err + } +} + +type errCache struct { + sync.Mutex + cache []regionErrorInfo + errCh chan regionErrorInfo + notify chan struct{} +} + +const errCacheDispatchBatchSize = 1024 + +func newErrCache() *errCache { + return &errCache{ + cache: make([]regionErrorInfo, 0, 1024), + errCh: make(chan regionErrorInfo, 4096), + notify: make(chan struct{}, 1024), + } +} + +func (e *errCache) add(errInfo regionErrorInfo) { + e.Lock() + defer e.Unlock() + e.cache = append(e.cache, errInfo) + select { + case e.notify <- struct{}{}: + default: + } +} + +func (e *errCache) popBatch(limit int) []regionErrorInfo { + e.Lock() + defer e.Unlock() + if len(e.cache) == 0 { + return nil + } + if limit <= 0 || limit > len(e.cache) { + limit = len(e.cache) + } + batch := make([]regionErrorInfo, limit) + copy(batch, e.cache[:limit]) + clear(e.cache[:limit]) + if limit == len(e.cache) { + e.cache = e.cache[:0] + } else { + e.cache = e.cache[limit:] + } + return batch +} + +func (e *errCache) dispatchBatch(ctx context.Context, limit int) (int, error) { + batch := e.popBatch(limit) + for _, errInfo := range batch { + select { + case <-ctx.Done(): + log.Info("subscription client dispatch err cache done") + return 0, ctx.Err() + case e.errCh <- errInfo: + } + } + return len(batch), nil +} + +func (e *errCache) dispatch(ctx context.Context) error { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + sendToErrCh := func() error { + for { + n, err := e.dispatchBatch(ctx, errCacheDispatchBatchSize) + if err != nil { + return err + } + if n < errCacheDispatchBatchSize { + return nil + } + } + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := sendToErrCh(); err != nil { + return err + } + case <-e.notify: + if err := sendToErrCh(); err != nil { + return err + } + } + } +} diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 9714938c20..a5272b0ddf 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -35,8 +35,8 @@ type regionRequestScheduler struct { config *SubscriptionClientConfig upstream *upstreamHandle - pushRegionEventToDS func(SubscriptionID, regionEvent) - onRegionFail func(regionErrorInfo) + eventSink *regionEventSink + failureReporter *regionFailureReporter queue *priorityqueue.PriorityQueue[*regionPriorityTask] seq atomic.Uint64 @@ -46,11 +46,11 @@ type regionRequestScheduler struct { func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { return ®ionRequestScheduler{ - config: client.config, - upstream: client.upstream, - pushRegionEventToDS: client.pushRegionEventToDS, - onRegionFail: client.onRegionFail, - queue: priorityqueue.New[*regionPriorityTask](), + config: client.config, + upstream: client.upstream, + eventSink: client.eventSink, + failureReporter: client.failureReporter, + queue: priorityqueue.New[*regionPriorityTask](), } } @@ -103,7 +103,7 @@ func (s *regionRequestScheduler) attachRPCContextForRegion(ctx context.Context, zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } - s.onRegionFail(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + s.failureReporter.Report(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) return region, false } @@ -150,8 +150,8 @@ func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) er rs, perWorkerQueueSize, s.upstream, - s.pushRegionEventToDS, - s.onRegionFail, + s.eventSink, + s.failureReporter, ) rs.requestWorkers = append(rs.requestWorkers, requestWorker) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index c72c77dfe1..f29a763644 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -82,9 +82,9 @@ type regionRequestWorker struct { store *requestedStore - upstream *upstreamHandle - pushRegionEventToDS func(SubscriptionID, regionEvent) - onRegionFail func(regionErrorInfo) + upstream *upstreamHandle + eventSink *regionEventSink + failureReporter *regionFailureReporter // we must always get a region to request before create a grpc stream. // only in this way we can avoid to try to connect to an offline store infinitely. @@ -108,15 +108,15 @@ func newRegionRequestWorker( store *requestedStore, requestCacheSize int, upstream *upstreamHandle, - pushRegionEventToDS func(SubscriptionID, regionEvent), - onRegionFail func(regionErrorInfo), + eventSink *regionEventSink, + failureReporter *regionFailureReporter, ) *regionRequestWorker { worker := ®ionRequestWorker{ - workerID: workerIDGen.Add(1), - store: store, - upstream: upstream, - pushRegionEventToDS: pushRegionEventToDS, - onRegionFail: onRegionFail, + workerID: workerIDGen.Add(1), + store: store, + upstream: upstream, + eventSink: eventSink, + failureReporter: failureReporter, requestCache: newRequestCache(requestCacheSize, func() { store.promoteDeferredTask() }), @@ -175,7 +175,7 @@ func newRegionRequestWorker( regionEvent := regionEvent{ states: []*regionFeedState{state}, } - worker.pushRegionEventToDS(subID, regionEvent) + worker.eventSink.Push(subID, regionEvent) } } // The store may fail forever, so we need try to re-schedule all pending regions. @@ -184,7 +184,7 @@ func newRegionRequestWorker( // It means it's a special task for stopping the table. continue } - worker.onRegionFail(newRegionErrorInfo(region, regionErr)) + worker.failureReporter.Report(newRegionErrorInfo(region, regionErr)) } if err := util.Hang(ctx, time.Second); err != nil { return err @@ -317,7 +317,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) default: log.Panic("unknown event type", zap.Any("event", event)) } - s.pushRegionEventToDS(subscriptionID, regionEvent) + s.eventSink.Push(subscriptionID, regionEvent) } else { switch event.Event.(type) { case *cdcpb.Event_Error: @@ -360,7 +360,7 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res if len(resolvedStates) == 0 { return } - s.pushRegionEventToDS(subscriptionID, regionEvent{ + s.eventSink.Push(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: resolvedStates, }) @@ -425,7 +425,7 @@ func (s *regionRequestWorker) processRegionSendTask( regionEvent := regionEvent{ states: []*regionFeedState{state}, } - s.pushRegionEventToDS(req.subID, regionEvent) + s.eventSink.Push(req.subID, regionEvent) } return nil } @@ -478,7 +478,7 @@ func (s *regionRequestWorker) processRegionSendTask( // It can be skipped directly because there must be no pending states from // the stopped subscribedTable, or the special singleRegionInfo for stopping // the table will be handled later. - s.onRegionFail(newRegionErrorInfo(region, &storeStreamErr{})) + s.failureReporter.Report(newRegionErrorInfo(region, &storeStreamErr{})) regionReq.finish() } else { state := newRegionFeedState(region, uint64(subID), s, regionReq) diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 54c28565c2..25d7816013 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -181,9 +181,8 @@ func (m *mockRegionEventDynamicStream) GetMetrics() dynstream.Metrics[int, Subsc func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mockRegionEventDynamicStream, *cdcpb.ResolvedTs) { ds := &mockRegionEventDynamicStream{} - client := &subscriptionClient{ds: ds} worker := ®ionRequestWorker{ - pushRegionEventToDS: client.pushRegionEventToDS, + eventSink: ®ionEventSink{ctx: context.Background(), ds: ds}, } worker.requestedRegions.subscriptions = map[SubscriptionID]regionFeedStates{ 1: make(regionFeedStates, regionCount), @@ -213,7 +212,7 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT return } states := resolvedStates - s.pushRegionEventToDS(subscriptionID, regionEvent{ + s.eventSink.Push(subscriptionID, regionEvent{ resolvedTs: resolvedTsEvent.Ts, states: states, }) @@ -328,12 +327,12 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, - upstream: &upstreamHandle{}, - pushRegionEventToDS: func(SubscriptionID, regionEvent) {}, - onRegionFail: func(regionErrorInfo) {}, + requestCache: newRequestCache(10), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + upstream: &upstreamHandle{}, + eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, + failureReporter: newRegionFailureReporter(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) @@ -380,12 +379,12 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, - upstream: &upstreamHandle{}, - pushRegionEventToDS: func(SubscriptionID, regionEvent) {}, - onRegionFail: func(regionErrorInfo) {}, + requestCache: newRequestCache(10), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + upstream: &upstreamHandle{}, + eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, + failureReporter: newRegionFailureReporter(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index a0e8d4ed5a..1267fde5a3 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -26,13 +26,11 @@ import ( "github.com/pingcap/ticdc/logservice/txnutil" "github.com/pingcap/ticdc/pkg/common" appcontext "github.com/pingcap/ticdc/pkg/common/context" - "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" - "github.com/pingcap/ticdc/utils/dynstream" "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" @@ -44,11 +42,6 @@ const ( // Maximum total sleep time(in ms), 20 seconds. tikvRequestMaxBackoff = 20000 - // TiCDC always interacts with region leader, every time something goes wrong, - // failed region will be reloaded via `BatchLoadRegionsWithKeyRange` API. So we - // don't need to force reload region anymore. - regionScheduleReload = false - loadRegionRetryInterval time.Duration = 100 * time.Millisecond resolveLockMinInterval time.Duration = 10 * time.Second resolveLockTickInterval time.Duration = 2 * time.Second @@ -56,18 +49,8 @@ const ( ) var ( - metricFeedNotLeaderCounter = metrics.EventFeedErrorCounter.WithLabelValues("NotLeader") - metricFeedEpochNotMatchCounter = metrics.EventFeedErrorCounter.WithLabelValues("EpochNotMatch") - metricFeedRegionNotFoundCounter = metrics.EventFeedErrorCounter.WithLabelValues("RegionNotFound") - metricFeedDuplicateRequestCounter = metrics.EventFeedErrorCounter.WithLabelValues("DuplicateRequest") - metricFeedUnknownErrorCounter = metrics.EventFeedErrorCounter.WithLabelValues("Unknown") - metricFeedRPCCtxUnavailable = metrics.EventFeedErrorCounter.WithLabelValues("RPCCtxUnavailable") - metricGetStoreErr = metrics.EventFeedErrorCounter.WithLabelValues("GetStoreErr") - metricStoreSendRequestErr = metrics.EventFeedErrorCounter.WithLabelValues("SendRequestToStore") - metricKvIsBusyCounter = metrics.EventFeedErrorCounter.WithLabelValues("KvIsBusy") - metricKvCongestedCounter = metrics.EventFeedErrorCounter.WithLabelValues("KvCongested") - metricResolveLockSuccessCounter = metrics.SubscriptionClientResolveLockCounter.WithLabelValues("success") - metricResolveLockFailureCounter = metrics.SubscriptionClientResolveLockCounter.WithLabelValues("failure") + metricResolveLockSuccessCounter = metrics.SubscriptionClientResolveLockCounter.WithLabelValues("success") + metricResolveLockFailureCounter = metrics.SubscriptionClientResolveLockCounter.WithLabelValues("failure") metricSubscriptionClientDSChannelSize = metrics.DynamicStreamEventChanSize.WithLabelValues("event-store") metricSubscriptionClientDSPendingQueueLen = metrics.DynamicStreamPendingQueueLen.WithLabelValues("event-store") @@ -188,11 +171,8 @@ type subscriptionClient struct { lockResolver txnutil.LockResolver - ds dynstream.DynamicStream[int, SubscriptionID, regionEvent, *subscribedSpan, *regionEventHandler] - // the following three fields are used to manage feedback from ds and notify other goroutines - mu sync.Mutex - cond *sync.Cond - paused atomic.Bool + eventSink *regionEventSink + failureReporter *regionFailureReporter totalSpans struct { sync.RWMutex @@ -207,9 +187,6 @@ type subscriptionClient struct { // The tasks will be handled in `handleResolveLockTasks` goroutine. resolveLockTaskCh chan resolveLockTask resolveLockRateLimiter *resolveLockRateLimiter - // errCh is used to receive region errors. - // The errors will be handled in `handleErrors` goroutine. - errCache *errCache } // NewSubscriptionClient creates a client. @@ -234,27 +211,17 @@ func NewSubscriptionClient( rangeTaskCh: make(chan rangeTask, 1024), resolveLockTaskCh: make(chan resolveLockTask, 1024), resolveLockRateLimiter: newResolveLockRateLimiter(), - errCache: newErrCache(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) subClient.totalSpans.spanMap = make(map[SubscriptionID]*subscribedSpan) - option := dynstream.NewOption() - // Note: it is max batch size of the kv sent from tikv(not committed rows) - option.BatchCount = 1024 - // TODO: Set `UseBuffer` to true until we refactor the `regionEventHandler.Handle` method so that it doesn't call any method of the dynamic stream. Currently, if `UseBuffer` is set to false, there will be a deadlock: - // ds.handleLoop fetch events from `ch` -> regionEventHandler.Handle -> ds.RemovePath -> send event to `ch` - option.UseBuffer = true - option.EnableMemoryControl = true - ds := dynstream.NewParallelDynamicStream( - "log-puller", - ®ionEventHandler{subClient: subClient}, - option, + subClient.failureReporter = newRegionFailureReporter( + subClient.upstream, + subClient.onTableDrained, + subClient.scheduleRegionRequest, + subClient.scheduleRangeRequest, ) - ds.Start() - subClient.ds = ds - subClient.cond = sync.NewCond(&subClient.mu) - + subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureReporter) subClient.regionScheduler = newRegionRequestScheduler(subClient) return subClient } @@ -280,7 +247,7 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { if resolvedTsLag > 0 { metrics.LogPullerResolvedTsLag.Set(resolvedTsLag) } - dsMetrics := s.ds.GetMetrics() + dsMetrics := s.eventSink.Metrics() metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) if len(dsMetrics.MemoryControl.AreaMemoryMetrics) > 1 { @@ -339,8 +306,7 @@ func (s *subscriptionClient) Subscribe( s.totalSpans.spanMap[subID] = rt s.totalSpans.Unlock() - areaSetting := dynstream.NewAreaSettingsWithMaxPendingSize(1*1024*1024*1024, dynstream.MemoryControlForPuller, "logPuller") // 1GB - s.ds.AddPath(rt.subID, rt, areaSetting) + s.eventSink.AddPath(rt) select { case <-s.ctx.Done(): @@ -370,56 +336,6 @@ func (s *subscriptionClient) Unsubscribe(subID SubscriptionID) { zap.Bool("exists", rt != nil)) } -func (s *subscriptionClient) wakeSubscription(subID SubscriptionID) { - s.ds.Wake(subID) -} - -func (s *subscriptionClient) pushRegionEventToDS(subID SubscriptionID, event regionEvent) { - // fast path - if !s.paused.Load() { - s.ds.Push(subID, event) - return - } - // slow path: wait until paused is false - s.mu.Lock() - for s.paused.Load() { - select { - case <-s.ctx.Done(): - s.mu.Unlock() - return - default: - s.cond.Wait() - } - } - s.mu.Unlock() - s.ds.Push(subID, event) -} - -func (s *subscriptionClient) handleDSFeedBack(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - case feedback := <-s.ds.Feedback(): - switch feedback.FeedbackType { - case dynstream.PauseArea: - s.mu.Lock() - s.paused.Store(true) - s.mu.Unlock() - log.Info("subscription client pause push region event") - case dynstream.ResumeArea: - s.mu.Lock() - s.paused.Store(false) - s.cond.Broadcast() - s.mu.Unlock() - log.Info("subscription client resume push region event") - case dynstream.ReleasePath, dynstream.ResumePath: - // Ignore it, because it is no need to pause and resume a path in puller. - } - } - } -} - func (s *subscriptionClient) Run(ctx context.Context) error { // s.consume = consume if s.upstream == nil || s.upstream.pd == nil { @@ -431,14 +347,13 @@ func (s *subscriptionClient) Run(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return s.updateMetrics(ctx) }) - g.Go(func() error { return s.handleDSFeedBack(ctx) }) + g.Go(func() error { return s.eventSink.Run(ctx) }) + g.Go(func() error { return s.failureReporter.Run(ctx) }) g.Go(func() error { return s.handleRangeTasks(ctx) }) g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) - g.Go(func() error { return s.handleErrors(ctx) }) g.Go(func() error { return s.runResolveLockChecker(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) g.Go(func() error { return s.logSlowRegions(ctx) }) - g.Go(func() error { return s.errCache.dispatch(ctx) }) log.Info("subscription client starts") defer log.Info("subscription client exits") @@ -448,11 +363,7 @@ func (s *subscriptionClient) Run(ctx context.Context) error { // Close closes the client. Must be called after `Run` returns. func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() - s.mu.Lock() - s.paused.Store(false) - s.cond.Broadcast() - s.mu.Unlock() - s.ds.Close() + s.eventSink.Close() if s.regionScheduler != nil { s.regionScheduler.close() } @@ -478,7 +389,7 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { log.Info("subscription client stop span is finished", zap.Uint64("subscriptionID", uint64(rt.subID))) - err := s.ds.RemovePath(rt.subID) + err := s.eventSink.RemovePath(rt.subID) if err != nil { log.Warn("subscription client remove path failed", zap.Uint64("subscriptionID", uint64(rt.subID)), @@ -489,18 +400,6 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { delete(s.totalSpans.spanMap, rt.subID) } -// Note: don't block the caller, otherwise there may be deadlock -func (s *subscriptionClient) onRegionFail(errInfo regionErrorInfo) { - // unlock the range early to prevent blocking the range. - if errInfo.subscribedSpan.rangeLock.UnlockRange( - errInfo.span.StartKey, errInfo.span.EndKey, - errInfo.verID.GetID(), errInfo.verID.GetVer(), errInfo.resolvedTs()) { - s.onTableDrained(errInfo.subscribedSpan) - return - } - s.errCache.add(errInfo) -} - func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { g, ctx := errgroup.WithContext(ctx) // Limit the concurrent number of goroutines to convert range tasks to region tasks. @@ -638,106 +537,6 @@ func (s *subscriptionClient) scheduleRangeRequest( } } -func (s *subscriptionClient) handleErrors(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - log.Info("subscription client handle errors and exit") - return ctx.Err() - case errInfo := <-s.errCache.errCh: - if err := s.doHandleError(ctx, errInfo); err != nil { - return err - } - } - } -} - -func (s *subscriptionClient) doHandleError(ctx context.Context, errInfo regionErrorInfo) error { - err := errors.Cause(errInfo.err) - if _, requestCancelled := err.(*requestCancelledErr); !requestCancelled { - log.Debug("cdc region error", - zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), - zap.Uint64("regionID", errInfo.verID.GetID()), - zap.Error(err)) - } - - //nolint:errorlint // converting large type switch to errors.As is a significant refactor - switch eerr := err.(type) { - case *eventError: - innerErr := eerr.err - if notLeader := innerErr.GetNotLeader(); notLeader != nil { - metricFeedNotLeaderCounter.Inc() - s.upstream.regionCache.UpdateLeader(errInfo.verID, notLeader.GetLeader(), errInfo.rpcCtx.AccessIdx) - s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) - return nil - } - if innerErr.GetEpochNotMatch() != nil { - metricFeedEpochNotMatchCounter.Inc() - s.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) - return nil - } - if innerErr.GetRegionNotFound() != nil { - metricFeedRegionNotFoundCounter.Inc() - s.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) - return nil - } - if innerErr.GetCongested() != nil { - metricKvCongestedCounter.Inc() - s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskLowPrior) - return nil - } - if innerErr.GetServerIsBusy() != nil { - metricKvIsBusyCounter.Inc() - s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskLowPrior) - return nil - } - if duplicated := innerErr.GetDuplicateRequest(); duplicated != nil { - // TODO(qupeng): It's better to add a new machanism to deregister one region. - metricFeedDuplicateRequestCounter.Inc() - return errors.New("duplicate request") - } - if compatibility := innerErr.GetCompatibility(); compatibility != nil { - return errors.ErrVersionIncompatible.GenWithStackByArgs(compatibility) - } - if mismatch := innerErr.GetClusterIdMismatch(); mismatch != nil { - return errors.ErrClusterIDMismatch.GenWithStackByArgs(mismatch.Current, mismatch.Request) - } - - log.Warn("empty or unknown cdc error", - zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), - zap.Stringer("error", innerErr)) - metricFeedUnknownErrorCounter.Inc() - s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) - return nil - case *rpcCtxUnavailableErr: - metricFeedRPCCtxUnavailable.Inc() - s.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) - return nil - case *getStoreErr: - metricGetStoreErr.Inc() - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - // cannot get the store the region belongs to, so we need to reload the region. - s.upstream.regionCache.OnSendFail(bo, errInfo.rpcCtx, true, err) - s.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) - return nil - case *storeStreamErr: - metricStoreSendRequestErr.Inc() - bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) - s.upstream.regionCache.OnSendFail(bo, errInfo.rpcCtx, regionScheduleReload, err) - s.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) - return nil - case *requestCancelledErr: - // the corresponding subscription has been unsubscribed, just ignore. - return nil - default: - // TODO(qupeng): for some errors it's better to just deregister the region from TiKVs. - log.Warn("subscription client meets an internal error, fail the changefeed", - zap.Uint64("subscriptionID", uint64(errInfo.subscribedSpan.subID)), - zap.Error(err)) - return err - } -} - type subscriptionAndTargetTs struct { subSpan *subscribedSpan targetTs uint64 @@ -958,93 +757,3 @@ func (r *subscribedSpan) resolveStaleLocks(targetTs uint64) { zap.Uint64("subscriptionID", uint64(r.subID)), zap.Any("ranges", res)) } - -type errCache struct { - sync.Mutex - cache []regionErrorInfo - errCh chan regionErrorInfo - notify chan struct{} -} - -const errCacheDispatchBatchSize = 1024 - -func newErrCache() *errCache { - return &errCache{ - cache: make([]regionErrorInfo, 0, 1024), - errCh: make(chan regionErrorInfo, 4096), - notify: make(chan struct{}, 1024), - } -} - -func (e *errCache) add(errInfo regionErrorInfo) { - e.Lock() - defer e.Unlock() - e.cache = append(e.cache, errInfo) - select { - case e.notify <- struct{}{}: - default: - } -} - -func (e *errCache) popBatch(limit int) []regionErrorInfo { - e.Lock() - defer e.Unlock() - if len(e.cache) == 0 { - return nil - } - if limit <= 0 || limit > len(e.cache) { - limit = len(e.cache) - } - batch := make([]regionErrorInfo, limit) - copy(batch, e.cache[:limit]) - clear(e.cache[:limit]) - if limit == len(e.cache) { - e.cache = e.cache[:0] - } else { - e.cache = e.cache[limit:] - } - return batch -} - -func (e *errCache) dispatchBatch(ctx context.Context, limit int) (int, error) { - batch := e.popBatch(limit) - for _, errInfo := range batch { - select { - case <-ctx.Done(): - log.Info("subscription client dispatch err cache done") - return 0, ctx.Err() - case e.errCh <- errInfo: - } - } - return len(batch), nil -} - -func (e *errCache) dispatch(ctx context.Context) error { - ticker := time.NewTicker(10 * time.Millisecond) - defer ticker.Stop() - sendToErrCh := func() error { - for { - n, err := e.dispatchBatch(ctx, errCacheDispatchBatchSize) - if err != nil { - return err - } - if n < errCacheDispatchBatchSize { - return nil - } - } - } - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - if err := sendToErrCh(); err != nil { - return err - } - case <-e.notify: - if err := sendToErrCh(); err != nil { - return err - } - } - } -} diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 1ad6dbd05f..52a133c65f 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -280,6 +280,7 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { client := &subscriptionClient{ resolveLockTaskCh: make(chan resolveLockTask, 1), upstream: &upstreamHandle{pdClock: pdutil.NewClock4Test()}, + eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, } client.ctx, client.cancel = context.WithCancel(context.Background()) defer client.cancel() @@ -312,9 +313,9 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { client := &subscriptionClient{ - errCache: newErrCache(), - ds: &mockDynamicStream{}, + eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, } + client.failureReporter = newRegionFailureReporter(&upstreamHandle{}, client.onTableDrained, nil, nil) rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte("a"), @@ -333,24 +334,24 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Equal(t, regionlock.LockRangeStatusSuccess, res2.Status) require.False(t, span.rangeLock.Stop()) - client.onRegionFail(newRegionErrorInfo(regionInfo{ + client.failureReporter.Report(newRegionErrorInfo(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("m")}, subscribedSpan: span, lockedRangeState: res1.LockedRangeState, }, &requestCancelledErr{})) - require.Len(t, client.errCache.cache, 1) + require.Len(t, client.failureReporter.cache.cache, 1) require.Len(t, span.rangeLock.IterAll(nil).UnLockedRanges, 1) - client.onRegionFail(newRegionErrorInfo(regionInfo{ + client.failureReporter.Report(newRegionErrorInfo(regionInfo{ verID: tikv.NewRegionVerID(2, 1, 1), span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("m"), EndKey: []byte("z")}, subscribedSpan: span, lockedRangeState: res2.LockedRangeState, }, &requestCancelledErr{})) - require.Len(t, client.errCache.cache, 1) + require.Len(t, client.failureReporter.cache.cache, 1) require.NotContains(t, client.totalSpans.spanMap, span.subID) } @@ -385,18 +386,16 @@ func (s *mockDynamicStream) GetMetrics() dynstream.Metrics[int, SubscriptionID] } func TestPushRegionEventToDSUnblocksOnClose(t *testing.T) { - client := &subscriptionClient{ - ds: &mockDynamicStream{}, - } - client.ctx, client.cancel = context.WithCancel(context.Background()) - client.cond = sync.NewCond(&client.mu) - client.regionScheduler = newRegionRequestScheduler(client) + ctx, cancel := context.WithCancel(context.Background()) + sink := ®ionEventSink{ctx: ctx, ds: &mockDynamicStream{}} + sink.cond = sync.NewCond(&sink.mu) + client := &subscriptionClient{cancel: cancel, eventSink: sink} - client.paused.Store(true) + sink.paused.Store(true) done := make(chan struct{}) go func() { - client.pushRegionEventToDS(SubscriptionID(1), regionEvent{}) + sink.Push(SubscriptionID(1), regionEvent{}) close(done) }() From 71e48c71fe79839af3afbe26c19f56630b4352c8 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 26 Jun 2026 15:58:00 -0700 Subject: [PATCH 14/21] fix --- logservice/logpuller/region_event_handler.go | 6 +- logservice/logpuller/region_event_sink.go | 4 +- ..._reporter.go => region_failure_handler.go} | 14 +- .../logpuller/region_request_scheduler.go | 18 +- logservice/logpuller/region_request_worker.go | 22 +-- .../logpuller/region_request_worker_test.go | 24 +-- logservice/logpuller/span_registry.go | 186 ++++++++++++++++++ logservice/logpuller/subscription_client.go | 186 +++--------------- .../logpuller/subscription_client_test.go | 21 +- 9 files changed, 263 insertions(+), 218 deletions(-) rename logservice/logpuller/{region_failure_reporter.go => region_failure_handler.go} (96%) create mode 100644 logservice/logpuller/span_registry.go diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 272bed7d9a..777af088bd 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -87,8 +87,8 @@ func (event regionEvent) mustFirstState() *regionFeedState { } type regionEventHandler struct { - eventSink *regionEventSink - failureReporter *regionFailureReporter + eventSink *regionEventSink + failureHandler *regionFailureHandler } func (h *regionEventHandler) Path(event regionEvent) SubscriptionID { @@ -256,7 +256,7 @@ func (h *regionEventHandler) handleRegionError(state *regionFeedState) { } if stepsToRemoved { worker.takeRegionState(SubscriptionID(state.requestID), state.getRegionID()) - h.failureReporter.Report(newRegionErrorInfo(state.getRegionInfo(), err)) + h.failureHandler.Report(newRegionErrorInfo(state.getRegionInfo(), err)) } } diff --git a/logservice/logpuller/region_event_sink.go b/logservice/logpuller/region_event_sink.go index 34a27dd177..0a90527d4b 100644 --- a/logservice/logpuller/region_event_sink.go +++ b/logservice/logpuller/region_event_sink.go @@ -32,7 +32,7 @@ type regionEventSink struct { paused atomic.Bool } -func newRegionEventSink(ctx context.Context, failureReporter *regionFailureReporter) *regionEventSink { +func newRegionEventSink(ctx context.Context, failureHandler *regionFailureHandler) *regionEventSink { sink := ®ionEventSink{ctx: ctx} sink.cond = sync.NewCond(&sink.mu) @@ -45,7 +45,7 @@ func newRegionEventSink(ctx context.Context, failureReporter *regionFailureRepor option.EnableMemoryControl = true ds := dynstream.NewParallelDynamicStream( "log-puller", - ®ionEventHandler{eventSink: sink, failureReporter: failureReporter}, + ®ionEventHandler{eventSink: sink, failureHandler: failureHandler}, option, ) ds.Start() diff --git a/logservice/logpuller/region_failure_reporter.go b/logservice/logpuller/region_failure_handler.go similarity index 96% rename from logservice/logpuller/region_failure_reporter.go rename to logservice/logpuller/region_failure_handler.go index 8ffe5271ab..f9b463d2cf 100644 --- a/logservice/logpuller/region_failure_reporter.go +++ b/logservice/logpuller/region_failure_handler.go @@ -46,7 +46,7 @@ var ( metricKvCongestedCounter = metrics.EventFeedErrorCounter.WithLabelValues("KvCongested") ) -type regionFailureReporter struct { +type regionFailureHandler struct { cache *errCache upstream *upstreamHandle @@ -55,13 +55,13 @@ type regionFailureReporter struct { scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, TaskType) } -func newRegionFailureReporter( +func newRegionFailureHandler( upstream *upstreamHandle, onTableDrained func(*subscribedSpan), scheduleRegionRequest func(context.Context, regionInfo, TaskType), scheduleRangeRequest func(context.Context, heartbeatpb.TableSpan, *subscribedSpan, bool, TaskType), -) *regionFailureReporter { - return ®ionFailureReporter{ +) *regionFailureHandler { + return ®ionFailureHandler{ cache: newErrCache(), upstream: upstream, onTableDrained: onTableDrained, @@ -73,7 +73,7 @@ func newRegionFailureReporter( // Report admits a region failure into the recovery pipeline. It releases the // corresponding range lock before enqueueing the failure so new range tasks are // not blocked by stale region ownership. -func (r *regionFailureReporter) Report(errInfo regionErrorInfo) { +func (r *regionFailureHandler) Report(errInfo regionErrorInfo) { if errInfo.subscribedSpan.rangeLock.UnlockRange( errInfo.span.StartKey, errInfo.span.EndKey, errInfo.verID.GetID(), errInfo.verID.GetVer(), errInfo.resolvedTs()) { @@ -83,7 +83,7 @@ func (r *regionFailureReporter) Report(errInfo regionErrorInfo) { r.cache.add(errInfo) } -func (r *regionFailureReporter) Run(ctx context.Context) error { +func (r *regionFailureHandler) Run(ctx context.Context) error { ticker := time.NewTicker(10 * time.Millisecond) defer ticker.Stop() handlePendingFailures := func() error { @@ -122,7 +122,7 @@ func (r *regionFailureReporter) Run(ctx context.Context) error { } } -func (r *regionFailureReporter) handleError(ctx context.Context, errInfo regionErrorInfo) error { +func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionErrorInfo) error { err := errors.Cause(errInfo.err) //nolint:errorlint // errors.Cause unwraps repository errors before the concrete type check. if _, requestCancelled := err.(*requestCancelledErr); !requestCancelled { diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index a5272b0ddf..ac7c9b6403 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -35,8 +35,8 @@ type regionRequestScheduler struct { config *SubscriptionClientConfig upstream *upstreamHandle - eventSink *regionEventSink - failureReporter *regionFailureReporter + eventSink *regionEventSink + failureHandler *regionFailureHandler queue *priorityqueue.PriorityQueue[*regionPriorityTask] seq atomic.Uint64 @@ -46,11 +46,11 @@ type regionRequestScheduler struct { func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { return ®ionRequestScheduler{ - config: client.config, - upstream: client.upstream, - eventSink: client.eventSink, - failureReporter: client.failureReporter, - queue: priorityqueue.New[*regionPriorityTask](), + config: client.config, + upstream: client.upstream, + eventSink: client.eventSink, + failureHandler: client.failureHandler, + queue: priorityqueue.New[*regionPriorityTask](), } } @@ -103,7 +103,7 @@ func (s *regionRequestScheduler) attachRPCContextForRegion(ctx context.Context, zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) } - s.failureReporter.Report(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) + s.failureHandler.Report(newRegionErrorInfo(region, &rpcCtxUnavailableErr{verID: region.verID})) return region, false } @@ -151,7 +151,7 @@ func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) er perWorkerQueueSize, s.upstream, s.eventSink, - s.failureReporter, + s.failureHandler, ) rs.requestWorkers = append(rs.requestWorkers, requestWorker) } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index f29a763644..69f6716a91 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -82,9 +82,9 @@ type regionRequestWorker struct { store *requestedStore - upstream *upstreamHandle - eventSink *regionEventSink - failureReporter *regionFailureReporter + upstream *upstreamHandle + eventSink *regionEventSink + failureHandler *regionFailureHandler // we must always get a region to request before create a grpc stream. // only in this way we can avoid to try to connect to an offline store infinitely. @@ -109,14 +109,14 @@ func newRegionRequestWorker( requestCacheSize int, upstream *upstreamHandle, eventSink *regionEventSink, - failureReporter *regionFailureReporter, + failureHandler *regionFailureHandler, ) *regionRequestWorker { worker := ®ionRequestWorker{ - workerID: workerIDGen.Add(1), - store: store, - upstream: upstream, - eventSink: eventSink, - failureReporter: failureReporter, + workerID: workerIDGen.Add(1), + store: store, + upstream: upstream, + eventSink: eventSink, + failureHandler: failureHandler, requestCache: newRequestCache(requestCacheSize, func() { store.promoteDeferredTask() }), @@ -184,7 +184,7 @@ func newRegionRequestWorker( // It means it's a special task for stopping the table. continue } - worker.failureReporter.Report(newRegionErrorInfo(region, regionErr)) + worker.failureHandler.Report(newRegionErrorInfo(region, regionErr)) } if err := util.Hang(ctx, time.Second); err != nil { return err @@ -478,7 +478,7 @@ func (s *regionRequestWorker) processRegionSendTask( // It can be skipped directly because there must be no pending states from // the stopped subscribedTable, or the special singleRegionInfo for stopping // the table will be handled later. - s.failureReporter.Report(newRegionErrorInfo(region, &storeStreamErr{})) + s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) regionReq.finish() } else { state := newRegionFeedState(region, uint64(subID), s, regionReq) diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 25d7816013..428556df27 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -327,12 +327,12 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, - upstream: &upstreamHandle{}, - eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, - failureReporter: newRegionFailureReporter(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), + requestCache: newRequestCache(10), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + upstream: &upstreamHandle{}, + eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, + failureHandler: newRegionFailureHandler(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) @@ -379,12 +379,12 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), - controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, - upstream: &upstreamHandle{}, - eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, - failureReporter: newRegionFailureReporter(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), + requestCache: newRequestCache(10), + controlQueue: newControlQueue(), + store: &requestedStore{storeAddr: "store-1"}, + upstream: &upstreamHandle{}, + eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, + failureHandler: newRegionFailureHandler(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) diff --git a/logservice/logpuller/span_registry.go b/logservice/logpuller/span_registry.go new file mode 100644 index 0000000000..baa04f7aa6 --- /dev/null +++ b/logservice/logpuller/span_registry.go @@ -0,0 +1,186 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/metrics" + "github.com/tikv/client-go/v2/oracle" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type spanRegistry struct { + sync.RWMutex + spans map[SubscriptionID]*subscribedSpan + + upstream *upstreamHandle +} + +func newSpanRegistry(upstream *upstreamHandle) *spanRegistry { + return &spanRegistry{ + spans: make(map[SubscriptionID]*subscribedSpan), + upstream: upstream, + } +} + +func (r *spanRegistry) Add(span *subscribedSpan) { + r.Lock() + defer r.Unlock() + r.spans[span.subID] = span +} + +func (r *spanRegistry) Get(subID SubscriptionID) *subscribedSpan { + r.RLock() + defer r.RUnlock() + return r.spans[subID] +} + +func (r *spanRegistry) Remove(subID SubscriptionID) { + r.Lock() + defer r.Unlock() + delete(r.spans, subID) +} + +func (r *spanRegistry) UpdateMetrics() { + count := 0 + pullerMinResolvedTs := uint64(0) + r.RLock() + for _, span := range r.spans { + count += span.rangeLock.Len() + resolvedTs := span.resolvedTs.Load() + if pullerMinResolvedTs == 0 || resolvedTs < pullerMinResolvedTs { + pullerMinResolvedTs = resolvedTs + } + } + r.RUnlock() + metrics.SubscriptionClientSubscribedRegionCount.Set(float64(count)) + + if pullerMinResolvedTs == 0 { + return + } + pdTime := r.upstream.pdClock.CurrentTime() + phyResolvedTs := oracle.ExtractPhysical(pullerMinResolvedTs) + resolvedTsLag := float64(oracle.GetPhysical(pdTime)-phyResolvedTs) / 1e3 + if resolvedTsLag > 0 { + metrics.LogPullerResolvedTsLag.Set(resolvedTsLag) + } +} + +func (r *spanRegistry) Run(ctx context.Context) error { + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { return r.runResolveLockChecker(ctx) }) + g.Go(func() error { return r.logSlowRegions(ctx) }) + return g.Wait() +} + +func (r *spanRegistry) runResolveLockChecker(ctx context.Context) error { + resolveLockTicker := time.NewTicker(resolveLockTickInterval) + defer resolveLockTicker.Stop() + maxCacheSize := 1024 + subSpanAndTsCache := make([]subscriptionAndTargetTs, 0, maxCacheSize) + // getResolvedTargetTs returns the targetTs to resolve stale locks. 0 means no need to resolve. + getResolvedTargetTs := func(subSpan *subscribedSpan, currentTime time.Time, currentTs uint64) uint64 { + resolvedTsUpdated := time.Unix(subSpan.resolvedTsUpdated.Load(), 0) + if !subSpan.initialized.Load() || time.Since(resolvedTsUpdated) < resolveLockFence { + return 0 + } + resolvedTs := subSpan.resolvedTs.Load() + resolvedTime := oracle.GetTimeFromTS(resolvedTs) + if currentTime.Sub(resolvedTime) < resolveLockFence { + return 0 + } + return min(currentTs, oracle.GoTimeToTS(resolvedTime.Add(resolveLockFence))) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-resolveLockTicker.C: + } + + physical, logic, err := r.upstream.pd.GetTS(ctx) + if err != nil { + log.Warn("get ts from pd failed", zap.Error(err)) + continue + } + currentTs := oracle.ComposeTS(physical, logic) + currentTime := r.upstream.pdClock.CurrentTime() + r.RLock() + for _, subSpan := range r.spans { + if subSpan != nil { + targetTs := getResolvedTargetTs(subSpan, currentTime, currentTs) + if targetTs > 0 { + subSpanAndTsCache = append(subSpanAndTsCache, subscriptionAndTargetTs{ + subSpan: subSpan, + targetTs: targetTs, + }) + } + } + } + r.RUnlock() + for _, subSpanAndTs := range subSpanAndTsCache { + subSpanAndTs.subSpan.resolveStaleLocks(subSpanAndTs.targetTs) + } + subSpanAndTsCache = subSpanAndTsCache[:0] + if cap(subSpanAndTsCache) > maxCacheSize { + subSpanAndTsCache = make([]subscriptionAndTargetTs, 0, maxCacheSize) + } + } +} + +func (r *spanRegistry) logSlowRegions(ctx context.Context) error { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + + currTime := r.upstream.pdClock.CurrentTime() + r.RLock() + for subscriptionID, rt := range r.spans { + attr := rt.rangeLock.IterAll(nil) + ckptTime := oracle.GetTimeFromTS(attr.SlowestRegion.ResolvedTs) + if attr.SlowestRegion.Initialized { + if currTime.Sub(ckptTime) > 6*resolveLockMinInterval { + log.Info("subscription client finds a initialized slow region", + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Any("slowRegion", attr.SlowestRegion)) + } + } else if currTime.Sub(attr.SlowestRegion.Created) > 10*time.Minute { + log.Info("subscription client initializes a region too slow", + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Any("slowRegion", attr.SlowestRegion)) + } else if currTime.Sub(ckptTime) > 10*time.Minute { + log.Info("subscription client finds a uninitialized slow region", + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Any("slowRegion", attr.SlowestRegion)) + } + if len(attr.UnLockedRanges) > 0 { + log.Info("subscription client holes exist", + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Any("holes", attr.UnLockedRanges)) + } + } + r.RUnlock() + } +} diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 1267fde5a3..08e3fa7901 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -15,7 +15,6 @@ package logpuller import ( "context" - "sync" "sync/atomic" "time" @@ -31,7 +30,6 @@ import ( "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/spanz" "github.com/pingcap/ticdc/pkg/util" - "github.com/tikv/client-go/v2/oracle" "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -143,6 +141,10 @@ type upstreamHandle struct { clusterID uint64 } +func (u *upstreamHandle) initialize(ctx context.Context) { + u.clusterID = u.pd.GetClusterID(ctx) +} + // subscriptionClient is used to subscribe events of table ranges from TiKV. // All exported Methods are thread-safe. type SubscriptionClient interface { @@ -171,13 +173,10 @@ type subscriptionClient struct { lockResolver txnutil.LockResolver - eventSink *regionEventSink - failureReporter *regionFailureReporter + eventSink *regionEventSink + failureHandler *regionFailureHandler - totalSpans struct { - sync.RWMutex - spanMap map[SubscriptionID]*subscribedSpan - } + spanRegistry *spanRegistry // rangeTaskCh is used to receive range tasks. // The tasks will be handled in `handleRangeTask` goroutine. @@ -213,15 +212,15 @@ func NewSubscriptionClient( resolveLockRateLimiter: newResolveLockRateLimiter(), } subClient.ctx, subClient.cancel = context.WithCancel(context.Background()) - subClient.totalSpans.spanMap = make(map[SubscriptionID]*subscribedSpan) + subClient.spanRegistry = newSpanRegistry(subClient.upstream) - subClient.failureReporter = newRegionFailureReporter( + subClient.failureHandler = newRegionFailureHandler( subClient.upstream, subClient.onTableDrained, subClient.scheduleRegionRequest, subClient.scheduleRangeRequest, ) - subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureReporter) + subClient.eventSink = newRegionEventSink(subClient.ctx, subClient.failureHandler) subClient.regionScheduler = newRegionRequestScheduler(subClient) return subClient } @@ -235,7 +234,7 @@ func (s *subscriptionClient) AllocSubscriptionID() SubscriptionID { return SubscriptionID(subscriptionIDGen.Add(1)) } -func (s *subscriptionClient) updateMetrics(ctx context.Context) error { +func (s *subscriptionClient) runMetricsUpdater(ctx context.Context) error { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { @@ -243,10 +242,6 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() case <-ticker.C: - resolvedTsLag := s.GetResolvedTsLag() - if resolvedTsLag > 0 { - metrics.LogPullerResolvedTsLag.Set(resolvedTsLag) - } dsMetrics := s.eventSink.Metrics() metricSubscriptionClientDSChannelSize.Set(float64(dsMetrics.EventChanSize)) metricSubscriptionClientDSPendingQueueLen.Set(float64(dsMetrics.PendingQueueLen)) @@ -270,21 +265,14 @@ func (s *subscriptionClient) updateMetrics(ctx context.Context) error { } s.regionScheduler.updateMetrics() - - count := 0 - s.totalSpans.RLock() - for _, rt := range s.totalSpans.spanMap { - count += rt.rangeLock.Len() - } - s.totalSpans.RUnlock() - metrics.SubscriptionClientSubscribedRegionCount.Set(float64(count)) + s.spanRegistry.UpdateMetrics() } } } // Subscribe the given table span. // NOTE: `span.TableID` must be set correctly. -// It new a subscribedSpan and store it in `s.totalSpans`, +// It new a subscribedSpan and store it in spanRegistry, // and send a rangeTask to `s.rangeTaskCh`. // The rangeTask will be handled in `handleRangeTasks` goroutine. func (s *subscriptionClient) Subscribe( @@ -302,9 +290,7 @@ func (s *subscriptionClient) Subscribe( } rt := s.newSubscribedSpan(subID, span, startTs, consumeKVEvents, advanceResolvedTs, advanceInterval, bdrMode) - s.totalSpans.Lock() - s.totalSpans.spanMap[subID] = rt - s.totalSpans.Unlock() + s.spanRegistry.Add(rt) s.eventSink.AddPath(rt) @@ -321,10 +307,8 @@ func (s *subscriptionClient) Subscribe( // Unsubscribe the given table span. All covered regions will be deregistered asynchronously. // NOTE: `span.TableID` must be set correctly. func (s *subscriptionClient) Unsubscribe(subID SubscriptionID) { - // NOTE: `subID` is cleared from `s.totalSpans` in `onTableDrained`. - s.totalSpans.Lock() - rt := s.totalSpans.spanMap[subID] - s.totalSpans.Unlock() + // NOTE: `subID` is cleared from `spanRegistry` in `onTableDrained`. + rt := s.spanRegistry.Get(subID) if rt == nil { log.Warn("unknown subscription", zap.Uint64("subscriptionID", uint64(subID))) return @@ -337,23 +321,16 @@ func (s *subscriptionClient) Unsubscribe(subID SubscriptionID) { } func (s *subscriptionClient) Run(ctx context.Context) error { - // s.consume = consume - if s.upstream == nil || s.upstream.pd == nil { - log.Warn("subscription client should be in test mode, skip run") - return nil - } - s.upstream.clusterID = s.upstream.pd.GetClusterID(ctx) + s.upstream.initialize(ctx) g, ctx := errgroup.WithContext(ctx) - - g.Go(func() error { return s.updateMetrics(ctx) }) g.Go(func() error { return s.eventSink.Run(ctx) }) - g.Go(func() error { return s.failureReporter.Run(ctx) }) - g.Go(func() error { return s.handleRangeTasks(ctx) }) + g.Go(func() error { return s.failureHandler.Run(ctx) }) g.Go(func() error { return s.regionScheduler.Run(ctx, g) }) - g.Go(func() error { return s.runResolveLockChecker(ctx) }) + g.Go(func() error { return s.handleRangeTasks(ctx) }) g.Go(func() error { return s.handleResolveLockTasks(ctx) }) - g.Go(func() error { return s.logSlowRegions(ctx) }) + g.Go(func() error { return s.spanRegistry.Run(ctx) }) + g.Go(func() error { return s.runMetricsUpdater(ctx) }) log.Info("subscription client starts") defer log.Info("subscription client exits") @@ -395,9 +372,7 @@ func (s *subscriptionClient) onTableDrained(rt *subscribedSpan) { zap.Uint64("subscriptionID", uint64(rt.subID)), zap.Error(err)) } - s.totalSpans.Lock() - defer s.totalSpans.Unlock() - delete(s.totalSpans.spanMap, rt.subID) + s.spanRegistry.Remove(rt.subID) } func (s *subscriptionClient) handleRangeTasks(ctx context.Context) error { @@ -542,62 +517,6 @@ type subscriptionAndTargetTs struct { targetTs uint64 } -func (s *subscriptionClient) runResolveLockChecker(ctx context.Context) error { - resolveLockTicker := time.NewTicker(resolveLockTickInterval) - defer resolveLockTicker.Stop() - maxCacheSize := 1024 - subSpanAndTsCache := make([]subscriptionAndTargetTs, 0, maxCacheSize) - // getResolvedTargetTs returns the targetTs to resolve stale locks. 0 means no need to resolve. - getResolvedTargetTs := func(subSpan *subscribedSpan, currentTime time.Time, currentTs uint64) uint64 { - resolvedTsUpdated := time.Unix(subSpan.resolvedTsUpdated.Load(), 0) - if !subSpan.initialized.Load() || time.Since(resolvedTsUpdated) < resolveLockFence { - return 0 - } - resolvedTs := subSpan.resolvedTs.Load() - resolvedTime := oracle.GetTimeFromTS(resolvedTs) - if currentTime.Sub(resolvedTime) < resolveLockFence { - return 0 - } - return min(currentTs, oracle.GoTimeToTS(resolvedTime.Add(resolveLockFence))) - } - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-resolveLockTicker.C: - } - - physical, logic, err := s.upstream.pd.GetTS(ctx) - if err != nil { - log.Warn("get ts from pd failed", zap.Error(err)) - continue - } - currentTs := oracle.ComposeTS(physical, logic) - currentTime := s.upstream.pdClock.CurrentTime() - s.totalSpans.Lock() - for _, subSpan := range s.totalSpans.spanMap { - if subSpan != nil { - targetTs := getResolvedTargetTs(subSpan, currentTime, currentTs) - if targetTs > 0 { - subSpanAndTsCache = append(subSpanAndTsCache, subscriptionAndTargetTs{ - subSpan: subSpan, - targetTs: targetTs, - }) - } - } - } - s.totalSpans.Unlock() - for _, subSpanAndTs := range subSpanAndTsCache { - subSpanAndTs.subSpan.resolveStaleLocks(subSpanAndTs.targetTs) - } - subSpanAndTsCache = subSpanAndTsCache[:0] - if cap(subSpanAndTsCache) > maxCacheSize { - subSpanAndTsCache = make([]subscriptionAndTargetTs, 0, maxCacheSize) - } - } -} - func (s *subscriptionClient) handleResolveLockTasks(ctx context.Context) error { doResolve := func(task resolveLockTask) { keyspaceID := task.keyspaceID @@ -636,48 +555,6 @@ func (s *subscriptionClient) handleResolveLockTasks(ctx context.Context) error { } } -func (s *subscriptionClient) logSlowRegions(ctx context.Context) error { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - } - - currTime := s.upstream.pdClock.CurrentTime() - s.totalSpans.RLock() - slowInitializeRegion := 0 - for subscriptionID, rt := range s.totalSpans.spanMap { - attr := rt.rangeLock.IterAll(nil) - ckptTime := oracle.GetTimeFromTS(attr.SlowestRegion.ResolvedTs) - if attr.SlowestRegion.Initialized { - if currTime.Sub(ckptTime) > 6*resolveLockMinInterval { - log.Info("subscription client finds a initialized slow region", - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Any("slowRegion", attr.SlowestRegion)) - } - } else if currTime.Sub(attr.SlowestRegion.Created) > 10*time.Minute { - slowInitializeRegion++ - log.Info("subscription client initializes a region too slow", - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Any("slowRegion", attr.SlowestRegion)) - } else if currTime.Sub(ckptTime) > 10*time.Minute { - log.Info("subscription client finds a uninitialized slow region", - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Any("slowRegion", attr.SlowestRegion)) - } - if len(attr.UnLockedRanges) > 0 { - log.Info("subscription client holes exist", - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Any("holes", attr.UnLockedRanges)) - } - } - s.totalSpans.RUnlock() - } -} - func (s *subscriptionClient) newSubscribedSpan( subID SubscriptionID, span heartbeatpb.TableSpan, @@ -731,25 +608,6 @@ func (s *subscriptionClient) newSubscribedSpan( return rt } -func (s *subscriptionClient) GetResolvedTsLag() float64 { - pullerMinResolvedTs := uint64(0) - s.totalSpans.RLock() - for _, rt := range s.totalSpans.spanMap { - resolvedTs := rt.resolvedTs.Load() - if pullerMinResolvedTs == 0 || resolvedTs < pullerMinResolvedTs { - pullerMinResolvedTs = resolvedTs - } - } - s.totalSpans.RUnlock() - if pullerMinResolvedTs == 0 { - return 0 - } - pdTime := s.upstream.pdClock.CurrentTime() - phyResolvedTs := oracle.ExtractPhysical(pullerMinResolvedTs) - lag := float64(oracle.GetPhysical(pdTime)-phyResolvedTs) / 1e3 - return lag -} - func (r *subscribedSpan) resolveStaleLocks(targetTs uint64) { util.MustCompareAndMonotonicIncrease(&r.staleLocksTargetTs, targetTs) res := r.rangeLock.IterAll(r.tryResolveLock) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 52a133c65f..d3a1177af2 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -60,6 +60,7 @@ func TestGenerateResolveLockTask(t *testing.T) { upstream: &upstreamHandle{pdClock: pdutil.NewClock4Test()}, } client.ctx, client.cancel = context.WithCancel(context.Background()) + client.spanRegistry = newSpanRegistry(client.upstream) rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte{'a'}, @@ -68,8 +69,7 @@ func TestGenerateResolveLockTask(t *testing.T) { consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} span := client.newSubscribedSpan(SubscriptionID(1), rawSpan, 100, consumeKVEvents, advanceResolvedTs, 0, false) - client.totalSpans.spanMap = make(map[SubscriptionID]*subscribedSpan) - client.totalSpans.spanMap[SubscriptionID(1)] = span + client.spanRegistry.Add(span) // Lock a range, and then ResolveLock will trigger a task for it. res := span.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) @@ -315,7 +315,8 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { client := &subscriptionClient{ eventSink: ®ionEventSink{ds: &mockDynamicStream{}}, } - client.failureReporter = newRegionFailureReporter(&upstreamHandle{}, client.onTableDrained, nil, nil) + client.spanRegistry = newSpanRegistry(&upstreamHandle{}) + client.failureHandler = newRegionFailureHandler(&upstreamHandle{}, client.onTableDrained, nil, nil) rawSpan := heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte("a"), @@ -326,7 +327,7 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { span: rawSpan, rangeLock: regionlock.NewRangeLock(1, rawSpan.StartKey, rawSpan.EndKey, 100), } - client.totalSpans.spanMap = map[SubscriptionID]*subscribedSpan{span.subID: span} + client.spanRegistry.Add(span) res1 := span.rangeLock.LockRange(context.Background(), []byte("a"), []byte("m"), 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res1.Status) @@ -334,25 +335,25 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Equal(t, regionlock.LockRangeStatusSuccess, res2.Status) require.False(t, span.rangeLock.Stop()) - client.failureReporter.Report(newRegionErrorInfo(regionInfo{ + client.failureHandler.Report(newRegionErrorInfo(regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("a"), EndKey: []byte("m")}, subscribedSpan: span, lockedRangeState: res1.LockedRangeState, }, &requestCancelledErr{})) - require.Len(t, client.failureReporter.cache.cache, 1) + require.Len(t, client.failureHandler.cache.cache, 1) require.Len(t, span.rangeLock.IterAll(nil).UnLockedRanges, 1) - client.failureReporter.Report(newRegionErrorInfo(regionInfo{ + client.failureHandler.Report(newRegionErrorInfo(regionInfo{ verID: tikv.NewRegionVerID(2, 1, 1), span: heartbeatpb.TableSpan{TableID: 1, StartKey: []byte("m"), EndKey: []byte("z")}, subscribedSpan: span, lockedRangeState: res2.LockedRangeState, }, &requestCancelledErr{})) - require.Len(t, client.failureReporter.cache.cache, 1) - require.NotContains(t, client.totalSpans.spanMap, span.subID) + require.Len(t, client.failureHandler.cache.cache, 1) + require.Nil(t, client.spanRegistry.Get(span.subID)) } type mockDynamicStream struct{} @@ -739,7 +740,7 @@ func TestGetResolvedTargetTs(t *testing.T) { }, 100, consumeKVEvents, advanceResolvedTs, 0, false) span.initialized.Store(true) - // Replicate the getResolvedTargetTs closure from runResolveLockChecker + // Replicate the getResolvedTargetTs closure from spanRegistry.runResolveLockChecker. getResolvedTargetTs := func(subSpan *subscribedSpan, currentTime time.Time, currentTs uint64) uint64 { resolvedTsUpdated := time.Unix(subSpan.resolvedTsUpdated.Load(), 0) if !subSpan.initialized.Load() || time.Since(resolvedTsUpdated) < resolveLockFence { From b152e048967c983ade24fe0432a56f63ce64adf2 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 26 Jun 2026 16:13:56 -0700 Subject: [PATCH 15/21] refactor --- logservice/logpuller/span_registry.go | 137 +++++++++++++++++- logservice/logpuller/subscription_client.go | 128 ++-------------- .../logpuller/subscription_client_test.go | 45 +++++- 3 files changed, 180 insertions(+), 130 deletions(-) diff --git a/logservice/logpuller/span_registry.go b/logservice/logpuller/span_registry.go index baa04f7aa6..c4cc080892 100644 --- a/logservice/logpuller/span_registry.go +++ b/logservice/logpuller/span_registry.go @@ -16,15 +16,59 @@ package logpuller import ( "context" "sync" + "sync/atomic" "time" "github.com/pingcap/log" + "github.com/pingcap/ticdc/heartbeatpb" + "github.com/pingcap/ticdc/logservice/logpuller/regionlock" + "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/pkg/util" "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) +const kvEventsCacheMaxSize = 32 + +// subscribedSpan is the local state for one subscribed table span. +type subscribedSpan struct { + subID SubscriptionID + startTs uint64 + // Whether to filter out the value written by TiCDC itself. + // It should be `true` in BDR mode. + filterLoop bool + + // The target span. + span heartbeatpb.TableSpan + // The range lock of the span, + // it is used to prevent duplicate requests to the same region range, + // and it also used to calculate this table's resolvedTs. + rangeLock *regionlock.RangeLock + + consumeKVEvents func(events []common.RawKVEntry, wakeCallback func()) bool + + advanceResolvedTs func(ts uint64) + + advanceInterval int64 + + kvEventsCache []common.RawKVEntry + + // To handle span removing. + stopped atomic.Bool + + // To handle stale lock resolvings. + tryResolveLock func(regionID uint64, state *regionlock.LockedRangeState) + staleLocksTargetTs atomic.Uint64 + + lastAdvanceTime atomic.Int64 + + initialized atomic.Bool + resolvedTsUpdated atomic.Int64 + resolvedTs atomic.Uint64 +} + type spanRegistry struct { sync.RWMutex spans map[SubscriptionID]*subscribedSpan @@ -32,6 +76,83 @@ type spanRegistry struct { upstream *upstreamHandle } +type spanAndTargetTs struct { + span *subscribedSpan + targetTs uint64 +} + +func newSubscribedSpan( + ctx context.Context, + resolveLockRateLimiter *resolveLockRateLimiter, + resolveLockTaskCh chan resolveLockTask, + subID SubscriptionID, + span heartbeatpb.TableSpan, + startTs uint64, + consumeKVEvents func(raw []common.RawKVEntry, wakeCallback func()) bool, + advanceResolvedTs func(ts uint64), + advanceInterval int64, + filterLoop bool, +) *subscribedSpan { + rangeLock := regionlock.NewRangeLock(uint64(subID), span.StartKey, span.EndKey, startTs) + + rt := &subscribedSpan{ + subID: subID, + span: span, + startTs: startTs, + filterLoop: filterLoop, + rangeLock: rangeLock, + + consumeKVEvents: consumeKVEvents, + advanceResolvedTs: advanceResolvedTs, + advanceInterval: advanceInterval, + } + rt.initialized.Store(false) + rt.resolvedTsUpdated.Store(time.Now().Unix()) + rt.resolvedTs.Store(startTs) + + rt.tryResolveLock = func(regionID uint64, state *regionlock.LockedRangeState) { + targetTs := rt.staleLocksTargetTs.Load() + if !state.Initialized.Load() || state.ResolvedTs.Load() >= targetTs { + return + } + key := resolveLockKey{keyspaceID: span.KeyspaceID, regionID: regionID} + if !resolveLockRateLimiter.trySchedule(key, time.Now()) { + return + } + select { + case <-ctx.Done(): + resolveLockRateLimiter.cancel(key) + case resolveLockTaskCh <- resolveLockTask{ + keyspaceID: span.KeyspaceID, + regionID: regionID, + targetTs: targetTs, + state: state, + }: + // it is ok to ignore resolve lock task when the channel is full + default: + resolveLockRateLimiter.cancel(key) + metrics.SubscriptionClientResolveLockTaskDropCounter.Inc() + } + } + return rt +} + +func (span *subscribedSpan) clearKVEventsCache() { + if cap(span.kvEventsCache) > kvEventsCacheMaxSize { + span.kvEventsCache = nil + } else { + span.kvEventsCache = span.kvEventsCache[:0] + } +} + +func (span *subscribedSpan) resolveStaleLocks(targetTs uint64) { + util.MustCompareAndMonotonicIncrease(&span.staleLocksTargetTs, targetTs) + res := span.rangeLock.IterAll(span.tryResolveLock) + log.Debug("subscription client finds slow locked ranges", + zap.Uint64("subscriptionID", uint64(span.subID)), + zap.Any("ranges", res)) +} + func newSpanRegistry(upstream *upstreamHandle) *spanRegistry { return &spanRegistry{ spans: make(map[SubscriptionID]*subscribedSpan), @@ -93,7 +214,7 @@ func (r *spanRegistry) runResolveLockChecker(ctx context.Context) error { resolveLockTicker := time.NewTicker(resolveLockTickInterval) defer resolveLockTicker.Stop() maxCacheSize := 1024 - subSpanAndTsCache := make([]subscriptionAndTargetTs, 0, maxCacheSize) + spanAndTsCache := make([]spanAndTargetTs, 0, maxCacheSize) // getResolvedTargetTs returns the targetTs to resolve stale locks. 0 means no need to resolve. getResolvedTargetTs := func(subSpan *subscribedSpan, currentTime time.Time, currentTs uint64) uint64 { resolvedTsUpdated := time.Unix(subSpan.resolvedTsUpdated.Load(), 0) @@ -127,20 +248,20 @@ func (r *spanRegistry) runResolveLockChecker(ctx context.Context) error { if subSpan != nil { targetTs := getResolvedTargetTs(subSpan, currentTime, currentTs) if targetTs > 0 { - subSpanAndTsCache = append(subSpanAndTsCache, subscriptionAndTargetTs{ - subSpan: subSpan, + spanAndTsCache = append(spanAndTsCache, spanAndTargetTs{ + span: subSpan, targetTs: targetTs, }) } } } r.RUnlock() - for _, subSpanAndTs := range subSpanAndTsCache { - subSpanAndTs.subSpan.resolveStaleLocks(subSpanAndTs.targetTs) + for _, spanAndTs := range spanAndTsCache { + spanAndTs.span.resolveStaleLocks(spanAndTs.targetTs) } - subSpanAndTsCache = subSpanAndTsCache[:0] - if cap(subSpanAndTsCache) > maxCacheSize { - subSpanAndTsCache = make([]subscriptionAndTargetTs, 0, maxCacheSize) + spanAndTsCache = spanAndTsCache[:0] + if cap(spanAndTsCache) > maxCacheSize { + spanAndTsCache = make([]spanAndTargetTs, 0, maxCacheSize) } } } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 08e3fa7901..12949451f5 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -79,55 +79,6 @@ type rangeTask struct { priority TaskType } -const kvEventsCacheMaxSize = 32 - -// subscribedSpan represents a span to subscribe. -// It contains a sub span of a table(or the total span of a table), -// the startTs of the table, and the output event channel. -type subscribedSpan struct { - subID SubscriptionID - startTs uint64 - // Whether to filter out the value written by TiCDC itself. - // It should be `true` in BDR mode. - filterLoop bool - - // The target span - span heartbeatpb.TableSpan - // The range lock of the span, - // it is used to prevent duplicate requests to the same region range, - // and it also used to calculate this table's resolvedTs. - rangeLock *regionlock.RangeLock - - consumeKVEvents func(events []common.RawKVEntry, wakeCallback func()) bool - - advanceResolvedTs func(ts uint64) - - advanceInterval int64 - - kvEventsCache []common.RawKVEntry - - // To handle span removing. - stopped atomic.Bool - - // To handle stale lock resolvings. - tryResolveLock func(regionID uint64, state *regionlock.LockedRangeState) - staleLocksTargetTs atomic.Uint64 - - lastAdvanceTime atomic.Int64 - - initialized atomic.Bool - resolvedTsUpdated atomic.Int64 - resolvedTs atomic.Uint64 -} - -func (span *subscribedSpan) clearKVEventsCache() { - if cap(span.kvEventsCache) > kvEventsCacheMaxSize { - span.kvEventsCache = nil - } else { - span.kvEventsCache = span.kvEventsCache[:0] - } -} - type SubscriptionClientConfig struct { // The number of region request workers to send region task for every tikv store RegionRequestWorkerPerStore uint @@ -289,7 +240,18 @@ func (s *subscriptionClient) Subscribe( return } - rt := s.newSubscribedSpan(subID, span, startTs, consumeKVEvents, advanceResolvedTs, advanceInterval, bdrMode) + rt := newSubscribedSpan( + s.ctx, + s.resolveLockRateLimiter, + s.resolveLockTaskCh, + subID, + span, + startTs, + consumeKVEvents, + advanceResolvedTs, + advanceInterval, + bdrMode, + ) s.spanRegistry.Add(rt) s.eventSink.AddPath(rt) @@ -512,11 +474,6 @@ func (s *subscriptionClient) scheduleRangeRequest( } } -type subscriptionAndTargetTs struct { - subSpan *subscribedSpan - targetTs uint64 -} - func (s *subscriptionClient) handleResolveLockTasks(ctx context.Context) error { doResolve := func(task resolveLockTask) { keyspaceID := task.keyspaceID @@ -554,64 +511,3 @@ func (s *subscriptionClient) handleResolveLockTasks(ctx context.Context) error { } } } - -func (s *subscriptionClient) newSubscribedSpan( - subID SubscriptionID, - span heartbeatpb.TableSpan, - startTs uint64, - consumeKVEvents func(raw []common.RawKVEntry, wakeCallback func()) bool, - advanceResolvedTs func(ts uint64), - advanceInterval int64, - filterLoop bool, -) *subscribedSpan { - rangeLock := regionlock.NewRangeLock(uint64(subID), span.StartKey, span.EndKey, startTs) - - rt := &subscribedSpan{ - subID: subID, - span: span, - startTs: startTs, - filterLoop: filterLoop, - rangeLock: rangeLock, - - consumeKVEvents: consumeKVEvents, - advanceResolvedTs: advanceResolvedTs, - advanceInterval: advanceInterval, - } - rt.initialized.Store(false) - rt.resolvedTsUpdated.Store(time.Now().Unix()) - rt.resolvedTs.Store(startTs) - - rt.tryResolveLock = func(regionID uint64, state *regionlock.LockedRangeState) { - targetTs := rt.staleLocksTargetTs.Load() - if !state.Initialized.Load() || state.ResolvedTs.Load() >= targetTs { - return - } - key := resolveLockKey{keyspaceID: span.KeyspaceID, regionID: regionID} - if !s.resolveLockRateLimiter.trySchedule(key, time.Now()) { - return - } - select { - case <-s.ctx.Done(): - s.resolveLockRateLimiter.cancel(key) - case s.resolveLockTaskCh <- resolveLockTask{ - keyspaceID: span.KeyspaceID, - regionID: regionID, - targetTs: targetTs, - state: state, - }: - // it is ok to ignore resolve lock task when the channel is full - default: - s.resolveLockRateLimiter.cancel(key) - metrics.SubscriptionClientResolveLockTaskDropCounter.Inc() - } - } - return rt -} - -func (r *subscribedSpan) resolveStaleLocks(targetTs uint64) { - util.MustCompareAndMonotonicIncrease(&r.staleLocksTargetTs, targetTs) - res := r.rangeLock.IterAll(r.tryResolveLock) - log.Debug("subscription client finds slow locked ranges", - zap.Uint64("subscriptionID", uint64(r.subID)), - zap.Any("ranges", res)) -} diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index d3a1177af2..5470a2f8af 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -68,7 +68,18 @@ func TestGenerateResolveLockTask(t *testing.T) { } consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} - span := client.newSubscribedSpan(SubscriptionID(1), rawSpan, 100, consumeKVEvents, advanceResolvedTs, 0, false) + span := newSubscribedSpan( + client.ctx, + client.resolveLockRateLimiter, + client.resolveLockTaskCh, + SubscriptionID(1), + rawSpan, + 100, + consumeKVEvents, + advanceResolvedTs, + 0, + false, + ) client.spanRegistry.Add(span) // Lock a range, and then ResolveLock will trigger a task for it. @@ -136,12 +147,12 @@ func TestResolveLockTaskDeduplicatedAcrossSubscribedSpans(t *testing.T) { consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} - span1 := client.newSubscribedSpan(SubscriptionID(1), heartbeatpb.TableSpan{ + span1 := newSubscribedSpan(client.ctx, client.resolveLockRateLimiter, client.resolveLockTaskCh, SubscriptionID(1), heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte{'a'}, EndKey: []byte{'z'}, }, 100, consumeKVEvents, advanceResolvedTs, 0, false) - span2 := client.newSubscribedSpan(SubscriptionID(2), heartbeatpb.TableSpan{ + span2 := newSubscribedSpan(client.ctx, client.resolveLockRateLimiter, client.resolveLockTaskCh, SubscriptionID(2), heartbeatpb.TableSpan{ TableID: 2, StartKey: []byte{'a'}, EndKey: []byte{'z'}, @@ -244,7 +255,18 @@ func TestResolveLockTaskDroppedWhenChannelFull(t *testing.T) { } consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} - span := client.newSubscribedSpan(SubscriptionID(1), rawSpan, 100, consumeKVEvents, advanceResolvedTs, 0, false) + span := newSubscribedSpan( + client.ctx, + client.resolveLockRateLimiter, + client.resolveLockTaskCh, + SubscriptionID(1), + rawSpan, + 100, + consumeKVEvents, + advanceResolvedTs, + 0, + false, + ) res := span.rangeLock.LockRange(context.Background(), []byte{'b'}, []byte{'c'}, 1, 100) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) @@ -293,7 +315,18 @@ func TestStopTaskUsesSubscribedSpanFilterLoop(t *testing.T) { } consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} - span := client.newSubscribedSpan(SubscriptionID(1), rawSpan, 100, consumeKVEvents, advanceResolvedTs, 0, true) + span := newSubscribedSpan( + client.ctx, + client.resolveLockRateLimiter, + client.resolveLockTaskCh, + SubscriptionID(1), + rawSpan, + 100, + consumeKVEvents, + advanceResolvedTs, + 0, + true, + ) res := span.rangeLock.LockRange(context.Background(), rawSpan.StartKey, rawSpan.EndKey, 1, 1) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) @@ -733,7 +766,7 @@ func TestGetResolvedTargetTs(t *testing.T) { consumeKVEvents := func(_ []common.RawKVEntry, _ func()) bool { return false } advanceResolvedTs := func(ts uint64) {} - span := client.newSubscribedSpan(SubscriptionID(1), heartbeatpb.TableSpan{ + span := newSubscribedSpan(client.ctx, client.resolveLockRateLimiter, client.resolveLockTaskCh, SubscriptionID(1), heartbeatpb.TableSpan{ TableID: 1, StartKey: []byte{'a'}, EndKey: []byte{'z'}, From 36dfde3afdc513c9e3450b5f56b190fc4f6811c3 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Fri, 26 Jun 2026 21:17:12 -0700 Subject: [PATCH 16/21] refactor --- logservice/logpuller/region_request_cache.go | 20 ++- .../logpuller/region_request_cache_test.go | 16 +-- .../logpuller/region_request_scheduler.go | 5 - logservice/logpuller/region_request_worker.go | 122 ++++++++---------- .../logpuller/region_request_worker_test.go | 10 +- logservice/logpuller/region_state.go | 5 - logservice/logpuller/subscription_client.go | 3 +- .../logpuller/subscription_client_test.go | 2 +- 8 files changed, 76 insertions(+), 107 deletions(-) diff --git a/logservice/logpuller/region_request_cache.go b/logservice/logpuller/region_request_cache.go index af6fd125f4..65ae06b3cb 100644 --- a/logservice/logpuller/region_request_cache.go +++ b/logservice/logpuller/region_request_cache.go @@ -119,19 +119,15 @@ type requestCache struct { onSpaceAvailable func() } -func newRequestCache(maxPendingCount int, onSpaceAvailable ...func()) *requestCache { - res := &requestCache{ - requests: make(map[*regionReq]struct{}), - ready: make([]*regionReq, 0, maxPendingCount), - maxPendingCount: maxPendingCount, - readyAvailable: make(chan struct{}, 1), - spaceAvailable: make(chan struct{}, 1), +func newRequestCache(maxPendingCount int, onSpaceAvailable func()) *requestCache { + return &requestCache{ + requests: make(map[*regionReq]struct{}), + ready: make([]*regionReq, 0, maxPendingCount), + maxPendingCount: maxPendingCount, + readyAvailable: make(chan struct{}, 1), + spaceAvailable: make(chan struct{}, 1), + onSpaceAvailable: onSpaceAvailable, } - if len(onSpaceAvailable) > 0 { - res.onSpaceAvailable = onSpaceAvailable[0] - } - - return res } // add admits a data request into the worker window. diff --git a/logservice/logpuller/region_request_cache_test.go b/logservice/logpuller/region_request_cache_test.go index 6f96a7e138..e0b57c8e52 100644 --- a/logservice/logpuller/region_request_cache_test.go +++ b/logservice/logpuller/region_request_cache_test.go @@ -42,7 +42,7 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { } func TestRequestCacheAdd_NormalCase(t *testing.T) { - cache := newRequestCache(10) + cache := newRequestCache(10, nil) ctx := context.Background() region := createTestRegionInfo(1, 1) @@ -61,7 +61,7 @@ func TestRequestCacheAdd_NormalCase(t *testing.T) { } func TestRequestCacheAdd_ForceFlag(t *testing.T) { - cache := newRequestCache(1) + cache := newRequestCache(1, nil) ctx := context.Background() // Fill up the cache @@ -98,7 +98,7 @@ func TestRequestCacheAdd_ForceFlag(t *testing.T) { } func TestRequestCacheAdd_ContextCancellation(t *testing.T) { - cache := newRequestCache(1) + cache := newRequestCache(1, nil) // Fill up the cache region1 := createTestRegionInfo(1, 1) @@ -119,7 +119,7 @@ func TestRequestCacheAdd_ContextCancellation(t *testing.T) { } func TestRequestCacheAdd_RetryLimitExceeded(t *testing.T) { - cache := newRequestCache(1) + cache := newRequestCache(1, nil) ctx := context.Background() // Fill up the cache @@ -136,7 +136,7 @@ func TestRequestCacheAdd_RetryLimitExceeded(t *testing.T) { } func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { - cache := newRequestCache(2) + cache := newRequestCache(2, nil) ctx := context.Background() // Fill up the cache @@ -174,7 +174,7 @@ func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { } func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { - cache := newRequestCache(10) + cache := newRequestCache(10, nil) ctx := context.Background() const numGoroutines = 5 @@ -205,7 +205,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { } func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testing.T) { - cache := newRequestCache(10) + cache := newRequestCache(10, nil) ctx := context.Background() region := createTestRegionInfo(1, 1) @@ -227,7 +227,7 @@ func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testi } func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { - cache := newRequestCache(10) + cache := newRequestCache(10, nil) ctx := context.Background() region := createTestRegionInfo(1, 1) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index ac7c9b6403..30464107e4 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -174,11 +174,6 @@ func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) er } region := regionTask.GetRegionInfo() - if region.isStopped() { - s.broadcastDeregister(region.subscribedSpan.subID, region.filterLoop) - continue - } - promotedStore := regionTask.deferredStore.Load() region, ok := s.attachRPCContextForRegion(ctx, region) if !ok { diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 69f6716a91..9b74e48941 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -130,18 +130,12 @@ func newRegionRequestWorker( zap.Uint64("workerID", worker.workerID), zap.String("addr", store.storeAddr)) } - for { - req, err := worker.requestCache.pop(ctx) - if err != nil { - return err - } - if req.regionInfo.isStopped() { - req.finish() - continue - } - worker.preFetchForConnecting = req - return nil + req, err := worker.requestCache.pop(ctx) + if err != nil { + return err } + worker.preFetchForConnecting = req + return nil } g.Go(func() error { @@ -180,10 +174,6 @@ func newRegionRequestWorker( } // The store may fail forever, so we need try to re-schedule all pending regions. for _, region := range worker.clearPendingRegions() { - if region.isStopped() { - // It means it's a special task for stopping the table. - continue - } worker.failureHandler.Report(newRegionErrorInfo(region, regionErr)) } if err := util.Hang(ctx, time.Second); err != nil { @@ -445,65 +435,59 @@ func (s *regionRequestWorker) processRegionSendTask( regionReq := s.preFetchForConnecting s.preFetchForConnecting = nil for { + if regionReq != nil { + region := regionReq.regionInfo + subID := region.subscribedSpan.subID + log.Debug("region request worker gets a singleRegionInfo", + zap.Uint64("workerID", s.workerID), + zap.Uint64("subscriptionID", uint64(subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", s.store.storeAddr), + zap.Bool("bdrMode", region.filterLoop)) + + if region.subscribedSpan.stopped.Load() { + // The subscription has been stopped before this queued request is sent. + s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) + regionReq.finish() + } else { + state := newRegionFeedState(region, uint64(subID), s, regionReq) + state.start() + s.addRegionState(subID, region.verID.GetID(), state) + // Mark the request as sent before sending it. + // Otherwise there is a race with the receiver goroutine: + // 1. addRegionState makes the region visible to error handling. + // 2. doSend sends the request. + // 3. the receiver goroutine may receive a region error immediately. + // 4. markStopped runs before markSent, so the request may be finished + // before it is marked as sent. + // 5. the sender goroutine then calls markSent and must not make the + // finished request live again. + // + // Tracking the request before Send keeps requestedRegions and + // request lifecycle visible in the same order and avoids leaving stale + // requests behind. + regionReq.markSent() + if err := doSend(s.createRegionRequest(region)); err != nil { + state.markStopped(err) + return err + } + } + regionReq = nil + continue + } + if err := drainControl(); err != nil { return err } - if regionReq == nil { - if regionReq = s.requestCache.tryPop(); regionReq != nil { - continue - } - select { - case <-s.controlQueue.notify: - continue - case <-s.requestCache.readyAvailable: - regionReq = s.requestCache.tryPop() - continue - case <-ctx.Done(): - return ctx.Err() - } + if regionReq = s.requestCache.tryPop(); regionReq != nil { + continue } - - region := regionReq.regionInfo - subID := region.subscribedSpan.subID - log.Debug("region request worker gets a singleRegionInfo", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", s.store.storeAddr), - zap.Bool("bdrMode", region.filterLoop)) - - if region.isStopped() { - regionReq.finish() - } else if region.subscribedSpan.stopped.Load() { - // It can be skipped directly because there must be no pending states from - // the stopped subscribedTable, or the special singleRegionInfo for stopping - // the table will be handled later. - s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) - regionReq.finish() - } else { - state := newRegionFeedState(region, uint64(subID), s, regionReq) - state.start() - s.addRegionState(subID, region.verID.GetID(), state) - // Mark the request as sent before sending it. - // Otherwise there is a race with the receiver goroutine: - // 1. addRegionState makes the region visible to error handling. - // 2. doSend sends the request. - // 3. the receiver goroutine may receive a region error immediately. - // 4. markStopped runs before markSent, so the request may be finished - // before it is marked as sent. - // 5. the sender goroutine then calls markSent and must not make the - // finished request live again. - // - // Tracking the request before Send keeps requestedRegions and - // request lifecycle visible in the same order and avoids leaving stale - // requests behind. - regionReq.markSent() - if err := doSend(s.createRegionRequest(region)); err != nil { - state.markStopped(err) - return err - } + select { + case <-s.controlQueue.notify: + case <-s.requestCache.readyAvailable: + case <-ctx.Done(): + return ctx.Err() } - regionReq = nil } } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 428556df27..7707441a91 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -78,7 +78,7 @@ func TestRegionStatesOperation(t *testing.T) { func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + requestCache: newRequestCache(10, nil), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) @@ -109,7 +109,7 @@ func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + requestCache: newRequestCache(10, nil), } ctx := context.Background() @@ -295,7 +295,7 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + requestCache: newRequestCache(10, nil), } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) @@ -327,7 +327,7 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + requestCache: newRequestCache(10, nil), controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, upstream: &upstreamHandle{}, @@ -379,7 +379,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ - requestCache: newRequestCache(10), + requestCache: newRequestCache(10, nil), controlQueue: newControlQueue(), store: &requestedStore{storeAddr: "store-1"}, upstream: &upstreamHandle{}, diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 9e157bd5d9..b73b36a05a 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -48,11 +48,6 @@ type regionInfo struct { filterLoop bool } -func (s *regionInfo) isStopped() bool { - // lockedRange only nil when the region's subscribedTable is stopped. - return s.lockedRangeState == nil -} - func newRegionInfo( verID tikv.RegionVerID, span heartbeatpb.TableSpan, diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index 12949451f5..ec263c7ccf 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -314,8 +314,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { zap.Uint64("subscriptionID", uint64(rt.subID))) // Set stopped to true so we can stop handling region events from the table. - // Then send a special singleRegionInfo to regionRouter to deregister the table - // from all TiKV instances. + // Then broadcast deregister requests to all region request workers. if rt.stopped.CompareAndSwap(false, true) { s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 5470a2f8af..92fa18f7b3 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -454,7 +454,7 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { scheduler := newRegionRequestScheduler(client) worker := ®ionRequestWorker{ - requestCache: newRequestCache(1), + requestCache: newRequestCache(1, nil), controlQueue: newControlQueue(), } store := &requestedStore{scheduler: scheduler, storeAddr: "store-1"} From 0ecb201d0805bc07b189e7bf7bcd6f20729bc80a Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 29 Jun 2026 14:19:21 -0700 Subject: [PATCH 17/21] refactor --- logservice/logpuller/errors.go | 11 + .../logpuller/region_failure_handler.go | 3 + logservice/logpuller/region_priority_task.go | 6 - logservice/logpuller/region_request_cache.go | 84 +-- .../logpuller/region_request_cache_test.go | 113 ++-- .../logpuller/region_request_scheduler.go | 493 +++++++++++------- logservice/logpuller/region_request_worker.go | 32 +- .../logpuller/region_request_worker_test.go | 44 +- logservice/logpuller/region_state.go | 6 +- logservice/logpuller/subscription_client.go | 14 +- .../logpuller/subscription_client_test.go | 108 +++- server/server.go | 3 +- 12 files changed, 556 insertions(+), 361 deletions(-) diff --git a/logservice/logpuller/errors.go b/logservice/logpuller/errors.go index ad3c48a093..9e27c975ae 100644 --- a/logservice/logpuller/errors.go +++ b/logservice/logpuller/errors.go @@ -38,6 +38,17 @@ func (e *rpcCtxUnavailableErr) Error() string { e.verID.GetID(), e.verID.GetVer(), e.verID.GetConfVer()) } +type rpcCtxChangedError struct { + verID tikv.RegionVerID + from string + to string +} + +func (e *rpcCtxChangedError) Error() string { + return fmt.Sprintf("rpcCtx for region %v changed from %s to %s. ver:%v, confver:%v", + e.verID.GetID(), e.from, e.to, e.verID.GetVer(), e.verID.GetConfVer()) +} + type getStoreErr struct{} func (e *getStoreErr) Error() string { return "get store error" } diff --git a/logservice/logpuller/region_failure_handler.go b/logservice/logpuller/region_failure_handler.go index f9b463d2cf..912b263378 100644 --- a/logservice/logpuller/region_failure_handler.go +++ b/logservice/logpuller/region_failure_handler.go @@ -184,6 +184,9 @@ func (r *regionFailureHandler) handleError(ctx context.Context, errInfo regionEr metricFeedRPCCtxUnavailable.Inc() r.scheduleRangeRequest(ctx, errInfo.span, errInfo.subscribedSpan, errInfo.filterLoop, TaskHighPrior) return nil + case *rpcCtxChangedError: + r.scheduleRegionRequest(ctx, errInfo.regionInfo, TaskHighPrior) + return nil case *getStoreErr: metricGetStoreErr.Inc() bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) diff --git a/logservice/logpuller/region_priority_task.go b/logservice/logpuller/region_priority_task.go index 8a6d8e645d..b88dba051f 100644 --- a/logservice/logpuller/region_priority_task.go +++ b/logservice/logpuller/region_priority_task.go @@ -15,7 +15,6 @@ package logpuller import ( "fmt" - "sync/atomic" "github.com/tikv/client-go/v2/oracle" ) @@ -52,11 +51,6 @@ type regionPriorityTask struct { seq uint64 regionInfo regionInfo heapIndex int // for heap.Item interface - - // deferredStore is set when a task deferred by one store is promoted back - // to the global priority queue. It lets the scheduler clear the store's - // promoted marker after the task is admitted, moved, or dropped. - deferredStore atomic.Pointer[requestedStore] } // newRegionPriorityTask creates a new priority task for region. diff --git a/logservice/logpuller/region_request_cache.go b/logservice/logpuller/region_request_cache.go index 65ae06b3cb..1bef819570 100644 --- a/logservice/logpuller/region_request_cache.go +++ b/logservice/logpuller/region_request_cache.go @@ -50,57 +50,57 @@ type regionReq struct { regionInfo regionInfo createTime time.Time - // cache is set by requestCache.add. It lets regionFeedState finish exactly + // quota is acquired before the request enters requestCache and released + // when the request leaves the worker window. + quota *regionRequestQuota + + // cache is set by requestCache.Add. It lets regionFeedState finish exactly // the request that created it instead of looking up by subID/regionID. cache *requestCache // stage is guarded by requestCache.mu. stage regionReqStage } -func newRegionReq(cache *requestCache, region regionInfo) *regionReq { +func newRegionReq(cache *requestCache, region regionInfo, quota *regionRequestQuota) *regionReq { return ®ionReq{ regionInfo: region, createTime: time.Now(), + quota: quota, cache: cache, stage: regionReqStageQueued, } } -func (r *regionReq) markSent() { - if r == nil || r.cache == nil { - return - } +func (r *regionReq) MarkSent() { r.cache.markSent(r) } -func (r *regionReq) resolve() bool { - if r == nil || r.cache == nil { - return false - } +func (r *regionReq) Resolve() bool { return r.cache.resolve(r) } -func (r *regionReq) finish() bool { - if r == nil || r.cache == nil { - return false - } +func (r *regionReq) Finish() bool { return r.cache.finish(r) } +func (r *regionReq) ReleaseQuota() { + r.quota.Release() +} + // requestCache manages worker-local data requests with flow control. // // requests is the source of truth for live requests. A request is inserted by -// add(), moves through queued/processing/sent, and is removed by resolve(), -// finish(), takeUnsentRegions(), or clear(). +// Add(), moves through queued/processing/sent, and is removed by resolve(), +// finish(), TakeUnsentRegions(), or Clear(). type requestCache struct { mu sync.Mutex // requests owns every live data request in this worker. Its length is the - // flow-control count used by add(), getPendingCount(), and metrics. + // flow-control count used by Add(), PendingCount(), and metrics. requests map[*regionReq]struct{} // ready is the FIFO list of queued requests. Entries already popped or - // removed are left as nil/stale and skipped by tryPop; readyIdx is the next + // removed are left as nil/stale and skipped by TryPop; readyIdx is the next // candidate index. compactReadyLocked occasionally drops skipped entries. ready []*regionReq readyIdx int @@ -109,10 +109,10 @@ type requestCache struct { // this limit, matching the old force behavior for high-priority data requests. maxPendingCount int - // readyAvailable wakes a worker blocked in pop() when a queued request is + // readyAvailable wakes a worker blocked in Pop() when a queued request is // appended. It is a level-trigger hint; callers must re-check ready under mu. readyAvailable chan struct{} - // spaceAvailable wakes add() when a live request leaves requests. + // spaceAvailable wakes Add() when a live request leaves requests. spaceAvailable chan struct{} // onSpaceAvailable lets the store-level deferred scheduler retry a task when // this worker frees request capacity. @@ -130,15 +130,17 @@ func newRequestCache(maxPendingCount int, onSpaceAvailable func()) *requestCache } } -// add admits a data request into the worker window. -func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) (bool, error) { +// Add admits a data request into the worker window. +func (c *requestCache) Add( + ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, +) (bool, error) { start := time.Now() ticker := time.NewTicker(addReqRetryInterval) defer ticker.Stop() retries := addReqRetryLimit for { - if c.tryAdd(region, force) { + if c.tryAdd(region, force, quota) { metrics.SubscriptionClientAddRegionRequestDuration.Observe(time.Since(start).Seconds()) return true, nil } @@ -156,7 +158,7 @@ func (c *requestCache) add(ctx context.Context, region regionInfo, force bool) ( } } -func (c *requestCache) tryAdd(region regionInfo, force bool) bool { +func (c *requestCache) tryAdd(region regionInfo, force bool, quota *regionRequestQuota) bool { notifyReady := false c.mu.Lock() @@ -171,24 +173,17 @@ func (c *requestCache) tryAdd(region regionInfo, force bool) bool { return false } - req := newRegionReq(c, region) + req := newRegionReq(c, region, quota) c.requests[req] = struct{}{} c.ready = append(c.ready, req) notifyReady = true return true } -func (c *requestCache) canAdd(force bool) bool { - c.mu.Lock() - defer c.mu.Unlock() - - return force || len(c.requests) < c.maxPendingCount -} - // pop takes the next queued request and moves it into processing state. -func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { +func (c *requestCache) Pop(ctx context.Context) (*regionReq, error) { for { - if req := c.tryPop(); req != nil { + if req := c.TryPop(); req != nil { return req, nil } @@ -200,7 +195,7 @@ func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { } } -func (c *requestCache) tryPop() *regionReq { +func (c *requestCache) TryPop() *regionReq { c.mu.Lock() defer c.mu.Unlock() @@ -244,7 +239,7 @@ func (c *requestCache) resolve(req *regionReq) bool { zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), zap.Uint64("regionID", req.regionInfo.verID.GetID()), zap.Float64("cost", cost), - zap.Int("pendingCount", c.getPendingCount())) + zap.Int("pendingCount", c.PendingCount())) metrics.RegionRequestFinishScanDuration.Observe(cost) return true } @@ -268,14 +263,16 @@ func (c *requestCache) remove(req *regionReq) bool { c.mu.Unlock() if removed { + req.ReleaseQuota() c.notifySpace() } return removed } -func (c *requestCache) takeUnsentRegions() []regionInfo { +func (c *requestCache) TakeUnsentRegions() []regionInfo { c.mu.Lock() regions := make([]regionInfo, 0, len(c.requests)) + removedReqs := make([]*regionReq, 0, len(c.requests)) removed := 0 for req := range c.requests { if req.stage == regionReqStageSent { @@ -284,6 +281,7 @@ func (c *requestCache) takeUnsentRegions() []regionInfo { regions = append(regions, req.regionInfo) if c.removeLocked(req) { removed++ + removedReqs = append(removedReqs, req) } } if removed > 0 { @@ -292,17 +290,22 @@ func (c *requestCache) takeUnsentRegions() []regionInfo { c.mu.Unlock() if removed > 0 { + for _, req := range removedReqs { + req.ReleaseQuota() + } c.notifySpace() } return regions } // clear removes all live requests and returns their regions. -func (c *requestCache) clear() []regionInfo { +func (c *requestCache) Clear() []regionInfo { c.mu.Lock() regions := make([]regionInfo, 0, len(c.requests)) + removedReqs := make([]*regionReq, 0, len(c.requests)) for req := range c.requests { regions = append(regions, req.regionInfo) + removedReqs = append(removedReqs, req) delete(c.requests, req) req.stage = regionReqStageFinished } @@ -312,13 +315,16 @@ func (c *requestCache) clear() []regionInfo { c.mu.Unlock() if removed > 0 { + for _, req := range removedReqs { + req.ReleaseQuota() + } c.notifySpace() } return regions } // getPendingCount returns the number of queued, processing and sent requests. -func (c *requestCache) getPendingCount() int { +func (c *requestCache) PendingCount() int { c.mu.Lock() defer c.mu.Unlock() return len(c.requests) diff --git a/logservice/logpuller/region_request_cache_test.go b/logservice/logpuller/region_request_cache_test.go index e0b57c8e52..e353cbee13 100644 --- a/logservice/logpuller/region_request_cache_test.go +++ b/logservice/logpuller/region_request_cache_test.go @@ -15,6 +15,7 @@ package logpuller import ( "context" + "sync/atomic" "testing" "time" @@ -41,19 +42,23 @@ func createTestRegionInfo(subID SubscriptionID, regionID uint64) regionInfo { return newRegionInfo(verID, span, nil, subscribedSpan, false) } +func testRegionRequestQuota() *regionRequestQuota { + return ®ionRequestQuota{release: func() {}} +} + func TestRequestCacheAdd_NormalCase(t *testing.T) { cache := newRequestCache(10, nil) ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region, false) + ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 1, cache.getPendingCount()) + require.Equal(t, 1, cache.PendingCount()) // Verify the request was added to the queue - req, err := cache.pop(ctx) + req, err := cache.Pop(ctx) require.NoError(t, err) require.NotNil(t, req) require.Equal(t, region.verID.GetID(), req.regionInfo.verID.GetID()) @@ -66,35 +71,35 @@ func TestRequestCacheAdd_ForceFlag(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region1, false) + ok, err := cache.Add(ctx, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) + require.Equal(t, 1, cache.PendingCount()) // Try to add another request without force - should fail due to retry limit region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx, region2, false) + ok, err = cache.Add(ctx, region2, false, testRegionRequestQuota()) require.False(t, ok) require.NoError(t, err) // With force=true, the request bypasses the live request limit. region3 := createTestRegionInfo(1, 3) - ok, err = cache.add(ctx, region3, true) + ok, err = cache.Add(ctx, region3, true, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) + require.Equal(t, 2, cache.PendingCount()) - req, err := cache.pop(ctx) + req, err := cache.Pop(ctx) require.NoError(t, err) require.NotNil(t, req) require.Equal(t, region1.verID.GetID(), req.regionInfo.verID.GetID()) require.Equal(t, region1.subscribedSpan.subID, req.regionInfo.subscribedSpan.subID) - require.Equal(t, 2, cache.getPendingCount()) - req.markSent() + require.Equal(t, 2, cache.PendingCount()) + req.MarkSent() // resolve region1 - req.resolve() - require.Equal(t, 1, cache.getPendingCount()) + req.Resolve() + require.Equal(t, 1, cache.PendingCount()) } func TestRequestCacheAdd_ContextCancellation(t *testing.T) { @@ -103,7 +108,7 @@ func TestRequestCacheAdd_ContextCancellation(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) ctx1 := context.Background() - ok, err := cache.add(ctx1, region1, false) + ok, err := cache.Add(ctx1, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) @@ -112,7 +117,7 @@ func TestRequestCacheAdd_ContextCancellation(t *testing.T) { cancel() // Cancel immediately region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx2, region2, false) + ok, err = cache.Add(ctx2, region2, false, testRegionRequestQuota()) require.False(t, ok) require.Error(t, err) require.Equal(t, context.Canceled, err) @@ -124,13 +129,13 @@ func TestRequestCacheAdd_RetryLimitExceeded(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region1, false) + ok, err := cache.Add(ctx, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) // Try to add another request - should eventually hit retry limit region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx, region2, false) + ok, err = cache.Add(ctx, region2, false, testRegionRequestQuota()) require.False(t, ok) require.NoError(t, err) } @@ -141,36 +146,36 @@ func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region1, false) + ok, err := cache.Add(ctx, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) + require.Equal(t, 1, cache.PendingCount()) region2 := createTestRegionInfo(1, 2) - ok, err = cache.add(ctx, region2, false) + ok, err = cache.Add(ctx, region2, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) + require.Equal(t, 2, cache.PendingCount()) // Pop a request and mark it as sent, then resolve it to free up space - req, err := cache.pop(ctx) + req, err := cache.Pop(ctx) require.NoError(t, err) require.NotNil(t, req) - require.Equal(t, 2, cache.getPendingCount()) // pop doesn't change pendingCount - req.markSent() - require.Equal(t, 2, cache.getPendingCount()) + require.Equal(t, 2, cache.PendingCount()) // pop doesn't change pendingCount + req.MarkSent() + require.Equal(t, 2, cache.PendingCount()) // Resolve the request to free up space - success := req.resolve() + success := req.Resolve() require.True(t, success) - require.Equal(t, 1, cache.getPendingCount()) + require.Equal(t, 1, cache.PendingCount()) // Now we should be able to add another request region3 := createTestRegionInfo(1, 3) - ok, err = cache.add(ctx, region3, false) + ok, err = cache.Add(ctx, region3, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) + require.Equal(t, 2, cache.PendingCount()) } func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { @@ -184,7 +189,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { for i := range numGoroutines { go func(id int) { region := createTestRegionInfo(SubscriptionID(id%3), uint64(id)) - ok, err := cache.add(ctx, region, false) + ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) done <- err @@ -201,7 +206,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { } } - require.Equal(t, numGoroutines, cache.getPendingCount()) + require.Equal(t, numGoroutines, cache.PendingCount()) } func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testing.T) { @@ -210,18 +215,18 @@ func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testi region := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region, false) + ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - ok, err = cache.add(ctx, region, false) + ok, err = cache.Add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.getPendingCount()) + require.Equal(t, 2, cache.PendingCount()) - req1, err := cache.pop(ctx) + req1, err := cache.Pop(ctx) require.NoError(t, err) - req2, err := cache.pop(ctx) + req2, err := cache.Pop(ctx) require.NoError(t, err) require.NotSame(t, req1, req2) } @@ -232,17 +237,39 @@ func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { region := createTestRegionInfo(1, 1) - ok, err := cache.add(ctx, region, false) + ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 1, cache.getPendingCount()) + require.Equal(t, 1, cache.PendingCount()) - req, err := cache.pop(ctx) + req, err := cache.Pop(ctx) require.NoError(t, err) - req.markSent() - require.Equal(t, 1, cache.getPendingCount()) + req.MarkSent() + require.Equal(t, 1, cache.PendingCount()) + + req.Finish() + require.Equal(t, 0, cache.PendingCount()) +} - req.finish() - require.Equal(t, 0, cache.getPendingCount()) +func TestRequestCacheFinishReleasesQuotaOnce(t *testing.T) { + cache := newRequestCache(10, nil) + ctx := context.Background() + region := createTestRegionInfo(1, 1) + var releaseCount atomic.Int32 + quota := ®ionRequestQuota{ + release: func() { + releaseCount.Add(1) + }, + } + + ok, err := cache.Add(ctx, region, false, quota) + require.True(t, ok) + require.NoError(t, err) + + req, err := cache.Pop(ctx) + require.NoError(t, err) + require.True(t, req.Finish()) + require.False(t, req.Finish()) + require.Equal(t, int32(1), releaseCount.Load()) } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 30464107e4..d69fea692c 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -19,9 +19,8 @@ import ( "sync/atomic" "github.com/pingcap/log" - "github.com/pingcap/ticdc/pkg/config" - "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/utils/chann" "github.com/pingcap/ticdc/utils/priorityqueue" kvclientv2 "github.com/tikv/client-go/v2/kv" "github.com/tikv/client-go/v2/tikv" @@ -38,38 +37,114 @@ type regionRequestScheduler struct { eventSink *regionEventSink failureHandler *regionFailureHandler + // queue stores newly submitted tasks before they are routed to a TiKV store. queue *priorityqueue.PriorityQueue[*regionPriorityTask] - seq atomic.Uint64 - + // seq is assigned to each task and used as a FIFO tie-breaker when + // multiple tasks have the same priority. + seq atomic.Uint64 + + // schedulerNotify wakes Run to re-check queue or storeAvailable. + schedulerNotify chan struct{} + // storeAvailable is an unbounded ready-store queue. A store is pushed here + // when its quota is released, so deferred tasks for that store can be + // retried without scanning all stores or dropping notifications. + storeAvailable *chann.UnlimitedChannel[*requestedStore, any] + + // stores maps TiKV store address to its scheduler-local state. stores sync.Map } func newRegionRequestScheduler(client *subscriptionClient) *regionRequestScheduler { return ®ionRequestScheduler{ - config: client.config, - upstream: client.upstream, - eventSink: client.eventSink, - failureHandler: client.failureHandler, - queue: priorityqueue.New[*regionPriorityTask](), + config: client.config, + upstream: client.upstream, + eventSink: client.eventSink, + failureHandler: client.failureHandler, + queue: priorityqueue.New[*regionPriorityTask](), + schedulerNotify: make(chan struct{}, 1), + storeAvailable: chann.NewUnlimitedChannelDefault[*requestedStore](), } } -func (s *regionRequestScheduler) submit(taskType TaskType, region regionInfo) { +// Run admits region tasks from two sources: new tasks from the global priority +// queue, and deferred tasks from stores whose quota has become available. +func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) error { + // Store creation is serialized by the single scheduler loop. + getStore := func(storeAddr string) *requestedStore { + var rs *requestedStore + if v, ok := s.stores.Load(storeAddr); ok { + rs = v.(*requestedStore) + return rs + } + + rs = newRequestedStore(ctx, eg, s, storeAddr) + s.stores.Store(storeAddr, rs) + return rs + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + // Run is the only consumer of storeAvailable, so Len > 0 means this + // GetWithContext will not block. + if s.storeAvailable.Len() > 0 { + store, ok, err := s.storeAvailable.GetWithContext(context.Background()) + if err != nil || !ok { + continue + } + store.MarkAvailableDequeued() + if err := s.handleDeferredTasks(ctx, store); err != nil { + return err + } + continue + } + + regionTask, ok := s.queue.TryPop() + if ok { + if err := s.handleNewTask(ctx, getStore, regionTask); err != nil { + return err + } + continue + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.schedulerNotify: + } + } +} + +func (s *regionRequestScheduler) Submit(taskType TaskType, region regionInfo) { task := newRegionPriorityTask(taskType, region, s.upstream.pdClock.CurrentTS(), s.seq.Add(1)) - s.queue.Push(task) + if s.queue.Push(task) { + s.notifyScheduler() + } } -func (s *regionRequestScheduler) close() { - s.queue.Close() - s.releaseAdmittedRegionRequests() +func (s *regionRequestScheduler) BroadcastDeregister(subID SubscriptionID, filterLoop bool) { + s.stores.Range(func(_ any, value any) bool { + rs := value.(*requestedStore) + for _, worker := range rs.requestWorkers { + worker.controlQueue.push(deregisterRequest{ + subID: subID, + filterLoop: filterLoop, + }) + } + return true + }) } -func (s *regionRequestScheduler) updateMetrics() { +func (s *regionRequestScheduler) UpdateMetrics() { count := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) for _, worker := range store.requestWorkers { - count += worker.requestCache.getPendingCount() + count += worker.requestCache.PendingCount() } return true }) @@ -77,19 +152,23 @@ func (s *regionRequestScheduler) updateMetrics() { Set(float64(count)) } -func (s *regionRequestScheduler) broadcastDeregister(subID SubscriptionID, filterLoop bool) { - s.stores.Range(func(_ any, value any) bool { - rs := value.(*requestedStore) - for _, worker := range rs.requestWorkers { - worker.controlQueue.push(deregisterRequest{ - subID: subID, - filterLoop: filterLoop, - }) - } +func (s *regionRequestScheduler) Close() { + s.queue.Close() + s.notifyScheduler() + s.stores.Range(func(_, value any) bool { + store := value.(*requestedStore) + store.Close() return true }) } +func (s *regionRequestScheduler) notifyScheduler() { + select { + case s.schedulerNotify <- struct{}{}: + default: + } +} + func (s *regionRequestScheduler) attachRPCContextForRegion(ctx context.Context, region regionInfo) (regionInfo, bool) { bo := tikv.NewBackoffer(ctx, tikvRequestMaxBackoff) rpcCtx, err := s.upstream.regionCache.GetTiKVRPCContext(bo, region.verID, kvclientv2.ReplicaReadLeader, 0) @@ -107,118 +186,145 @@ func (s *regionRequestScheduler) attachRPCContextForRegion(ctx context.Context, return region, false } -// releaseAdmittedRegionRequests releases region requests admitted to store -// workers during scheduler shutdown. -func (s *regionRequestScheduler) releaseAdmittedRegionRequests() { - s.stores.Range(func(_, value any) bool { - rs := value.(*requestedStore) - rs.releaseAdmittedRegionRequests() - return true - }) -} - -// Run receives region tasks from the global priority queue and admits them to -// the corresponding TiKV store worker. -func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) error { - cfg := config.GetGlobalServerConfig() - pendingRegionRequestQueueSize := cfg.Debug.Puller.PendingRegionRequestQueueSize - // Store creation is serialized by the single scheduler loop. - getStore := func(storeAddr string) *requestedStore { - var rs *requestedStore - if v, ok := s.stores.Load(storeAddr); ok { - rs = v.(*requestedStore) - return rs - } - - perWorkerQueueSize := pendingRegionRequestQueueSize / int(s.config.RegionRequestWorkerPerStore) - if perWorkerQueueSize <= 0 { - log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", - zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), - zap.Uint("regionRequestWorkerPerStore", s.config.RegionRequestWorkerPerStore)) - perWorkerQueueSize = 1 - } - - rs = &requestedStore{ - scheduler: s, - storeAddr: storeAddr, - requestWorkers: make([]*regionRequestWorker, 0, s.config.RegionRequestWorkerPerStore), - } - for i := uint(0); i < s.config.RegionRequestWorkerPerStore; i++ { - requestWorker := newRegionRequestWorker( - ctx, - eg, - rs, - perWorkerQueueSize, - s.upstream, - s.eventSink, - s.failureHandler, - ) - rs.requestWorkers = append(rs.requestWorkers, requestWorker) - } - s.stores.Store(storeAddr, rs) - return rs - } +type getRequestedStoreFunc func(storeAddr string) *requestedStore +func (s *regionRequestScheduler) handleDeferredTasks(ctx context.Context, store *requestedStore) error { for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - regionTask, err := s.queue.Pop(ctx) - if err != nil { - if errors.Is(err, priorityqueue.ErrClosed) { - return nil - } - return err + task, ok := store.pendingTasks.TryPop() + if !ok { + return nil } - region := regionTask.GetRegionInfo() - promotedStore := regionTask.deferredStore.Load() - region, ok := s.attachRPCContextForRegion(ctx, region) + region, ok := s.attachRPCContextForRegion(ctx, task.GetRegionInfo()) if !ok { - if promotedStore != nil { - promotedStore.finishPromotedTask(regionTask) - promotedStore.promoteDeferredTask() - } continue } - - store := getStore(region.rpcCtx.Addr) - if promotedStore != nil && promotedStore != store { - promotedStore.finishPromotedTask(regionTask) - promotedStore.promoteDeferredTask() - } - if store.hasDeferredTaskAhead(regionTask) { - store.deferTask(regionTask) - store.maybePromoteDeferredTask() + task.regionInfo = region + + if region.rpcCtx.Addr != store.storeAddr { + s.failureHandler.Report(newRegionErrorInfo(region, &rpcCtxChangedError{ + verID: region.verID, + from: store.storeAddr, + to: region.rpcCtx.Addr, + })) continue } - force := regionTask.Priority() <= forcedPriorityBase - var worker *regionRequestWorker - ok, worker, err = store.addRegion(ctx, region, force) + ok, err := s.tryAdmitTask(ctx, store, task, region) if err != nil { - log.Warn("subscription client add region request failed", - zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), - zap.Uint64("regionID", region.verID.GetID()), - zap.Error(err)) return err } - if !ok { - store.deferTask(regionTask) - store.maybePromoteDeferredTask() - continue + store.pendingTasks.Push(task) + return nil } - store.finishPromotedTask(regionTask) + } +} - log.Debug("subscription client will request a region", - zap.Uint64("workID", worker.workerID), +func (s *regionRequestScheduler) handleNewTask( + ctx context.Context, + getStore getRequestedStoreFunc, + task *regionPriorityTask, +) error { + region, ok := s.attachRPCContextForRegion(ctx, task.GetRegionInfo()) + if !ok { + return nil + } + task.regionInfo = region + + store := getStore(region.rpcCtx.Addr) + if store.pendingTasks.Len() > 0 { + store.pendingTasks.Push(task) + store.NotifyAvailable() + return nil + } + + ok, err := s.tryAdmitTask(ctx, store, task, region) + if err != nil { + return err + } + if !ok { + store.pendingTasks.Push(task) + } + return nil +} + +func (s *regionRequestScheduler) tryAdmitTask( + ctx context.Context, + store *requestedStore, + task *regionPriorityTask, + region regionInfo, +) (bool, error) { + force := task.Priority() <= forcedPriorityBase + acquiredQuota, ok := store.quota.TryAcquire() + if !ok { + return false, nil + } + ok, worker, err := store.AddRegion(ctx, region, force, acquiredQuota) + if err != nil { + acquiredQuota.Release() + log.Warn("subscription client add region request failed", zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", store.storeAddr)) + zap.Error(err)) + return false, err + } + if !ok { + acquiredQuota.Release() + return false, nil + } + + log.Debug("subscription client will request a region", + zap.Uint64("workID", worker.workerID), + zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), + zap.Uint64("regionID", region.verID.GetID()), + zap.String("addr", store.storeAddr)) + return true, nil +} + +type regionRequestQuota struct { + once sync.Once + release func() +} + +func (p *regionRequestQuota) Release() { + p.once.Do(p.release) +} + +type storeQuota struct { + mu sync.Mutex + capacity int + used int + + onRelease func() +} + +func newStoreQuota(capacity int, onRelease func()) *storeQuota { + return &storeQuota{ + capacity: capacity, + onRelease: onRelease, + } +} + +func (q *storeQuota) TryAcquire() (*regionRequestQuota, bool) { + q.mu.Lock() + if q.used >= q.capacity { + q.mu.Unlock() + return nil, false } + q.used++ + q.mu.Unlock() + + return ®ionRequestQuota{ + release: func() { + q.mu.Lock() + q.used-- + q.mu.Unlock() + if q.onRelease != nil { + q.onRelease() + } + }, + }, true } // requestedStore is the scheduler's local state for one TiKV store. @@ -226,31 +332,76 @@ type requestedStore struct { scheduler *regionRequestScheduler storeAddr string + quota *storeQuota // nextWorker is the round-robin cursor used to select the next worker to attempt. nextWorker atomic.Uint32 // requestWorkers are fully created before requestedStore is published // and remain immutable afterwards. requestWorkers []*regionRequestWorker - deferredTasks struct { - sync.Mutex - // tasks are blocked because this store currently has no worker capacity. - tasks []*regionPriorityTask - // promoted is the deferred task that has been pushed back to the global - // queue but has not been admitted by this store yet. - promoted *regionPriorityTask + // pendingTasks holds tasks that have been routed to this store but are + // waiting for store quota or worker request-cache capacity. It is mutated + // only by the single regionRequestScheduler.Run loop. + pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] + + notifyMu sync.Mutex + notifyEnqueued bool +} + +func newRequestedStore( + ctx context.Context, + eg *errgroup.Group, + scheduler *regionRequestScheduler, + storeAddr string, +) *requestedStore { + pendingRegionRequestQueueSize := scheduler.config.PendingRegionRequestQueueSize + regionRequestWorkerPerStore := scheduler.config.RegionRequestWorkerPerStore + perWorkerQueueSize := pendingRegionRequestQueueSize / int(regionRequestWorkerPerStore) + if perWorkerQueueSize <= 0 { + log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", + zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), + zap.Uint("regionRequestWorkerPerStore", regionRequestWorkerPerStore)) + perWorkerQueueSize = 1 + } + perStoreQuotaSize := perWorkerQueueSize * int(regionRequestWorkerPerStore) + + rs := &requestedStore{ + scheduler: scheduler, + storeAddr: storeAddr, + requestWorkers: make([]*regionRequestWorker, 0, regionRequestWorkerPerStore), + pendingTasks: priorityqueue.New[*regionPriorityTask](), + } + rs.quota = newStoreQuota(perStoreQuotaSize, rs.NotifyAvailable) + for range regionRequestWorkerPerStore { + requestWorker := newRegionRequestWorker( + ctx, + eg, + rs, + perWorkerQueueSize, + scheduler.upstream, + scheduler.eventSink, + scheduler.failureHandler, + ) + rs.requestWorkers = append(rs.requestWorkers, requestWorker) + } + return rs +} + +func (rs *requestedStore) Close() { + for _, worker := range rs.requestWorkers { + worker.ReleaseAdmittedRegionRequests() } } -func (rs *requestedStore) addRegion( - ctx context.Context, region regionInfo, force bool, +func (rs *requestedStore) AddRegion( + ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, ) (bool, *regionRequestWorker, error) { workers := rs.requestWorkers start := int(rs.nextWorker.Add(1)) % len(workers) for i := range len(workers) { worker := workers[(start+i)%len(workers)] - ok, err := worker.add(ctx, region, force) + ok, err := worker.Add(ctx, region, force, quota) if err != nil || ok { return ok, worker, err } @@ -258,83 +409,21 @@ func (rs *requestedStore) addRegion( return false, nil, nil } -// hasDeferredTaskAhead reports whether an earlier deferred task should be -// retried before task for this store. -func (rs *requestedStore) hasDeferredTaskAhead(task *regionPriorityTask) bool { - rs.deferredTasks.Lock() - defer rs.deferredTasks.Unlock() - - if rs.deferredTasks.promoted == task { - // The promoted task was already removed from the deferred queue and is - // being retried through the global priority queue. - return false - } - return rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) > 0 -} - -func (rs *requestedStore) deferTask(task *regionPriorityTask) { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted == task { - rs.deferredTasks.promoted = nil - task.deferredStore.Store(nil) - rs.deferredTasks.tasks = append([]*regionPriorityTask{task}, rs.deferredTasks.tasks...) - } else { - rs.deferredTasks.tasks = append(rs.deferredTasks.tasks, task) - } - rs.deferredTasks.Unlock() -} - -func (rs *requestedStore) maybePromoteDeferredTask() { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { - rs.deferredTasks.Unlock() +func (rs *requestedStore) NotifyAvailable() { + rs.notifyMu.Lock() + if rs.notifyEnqueued { + rs.notifyMu.Unlock() return } - force := rs.deferredTasks.tasks[0].Priority() <= forcedPriorityBase - rs.deferredTasks.Unlock() - - if rs.hasRequestCapacity(force) { - rs.promoteDeferredTask() - } -} - -func (rs *requestedStore) hasRequestCapacity(force bool) bool { - for _, worker := range rs.requestWorkers { - if worker.requestCache.canAdd(force) { - return true - } - } - return false -} - -func (rs *requestedStore) releaseAdmittedRegionRequests() { - for _, worker := range rs.requestWorkers { - worker.releaseAdmittedRegionRequests() - } -} + rs.notifyEnqueued = true + rs.notifyMu.Unlock() -func (rs *requestedStore) finishPromotedTask(task *regionPriorityTask) { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted == task { - rs.deferredTasks.promoted = nil - task.deferredStore.Store(nil) - } - rs.deferredTasks.Unlock() + rs.scheduler.storeAvailable.Push(rs) + rs.scheduler.notifyScheduler() } -func (rs *requestedStore) promoteDeferredTask() { - rs.deferredTasks.Lock() - if rs.deferredTasks.promoted != nil || len(rs.deferredTasks.tasks) == 0 { - rs.deferredTasks.Unlock() - return - } - task := rs.deferredTasks.tasks[0] - copy(rs.deferredTasks.tasks, rs.deferredTasks.tasks[1:]) - rs.deferredTasks.tasks[len(rs.deferredTasks.tasks)-1] = nil - rs.deferredTasks.tasks = rs.deferredTasks.tasks[:len(rs.deferredTasks.tasks)-1] - rs.deferredTasks.promoted = task - task.deferredStore.Store(rs) - rs.deferredTasks.Unlock() - - rs.scheduler.queue.Push(task) +func (rs *requestedStore) MarkAvailableDequeued() { + rs.notifyMu.Lock() + rs.notifyEnqueued = false + rs.notifyMu.Unlock() } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 9b74e48941..5a0e887dbf 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -118,7 +118,7 @@ func newRegionRequestWorker( eventSink: eventSink, failureHandler: failureHandler, requestCache: newRequestCache(requestCacheSize, func() { - store.promoteDeferredTask() + store.NotifyAvailable() }), controlQueue: newControlQueue(), } @@ -130,7 +130,7 @@ func newRegionRequestWorker( zap.Uint64("workerID", worker.workerID), zap.String("addr", store.storeAddr)) } - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.Pop(ctx) if err != nil { return err } @@ -158,7 +158,7 @@ func newRegionRequestWorker( regionErr = &storeStreamErr{} } } else { - if canceled := worker.run(ctx, worker.upstream.credential); canceled { + if canceled := worker.Run(ctx, worker.upstream.credential); canceled { return nil } regionErr = &storeStreamErr{} @@ -185,7 +185,7 @@ func newRegionRequestWorker( return worker } -func (s *regionRequestWorker) run(ctx context.Context, credential *security.Credential) (canceled bool) { +func (s *regionRequestWorker) Run(ctx context.Context, credential *security.Credential) (canceled bool) { isCanceled := func() bool { select { case <-ctx.Done(): @@ -448,7 +448,7 @@ func (s *regionRequestWorker) processRegionSendTask( if region.subscribedSpan.stopped.Load() { // The subscription has been stopped before this queued request is sent. s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) - regionReq.finish() + regionReq.Finish() } else { state := newRegionFeedState(region, uint64(subID), s, regionReq) state.start() @@ -466,7 +466,7 @@ func (s *regionRequestWorker) processRegionSendTask( // Tracking the request before Send keeps requestedRegions and // request lifecycle visible in the same order and avoids leaving stale // requests behind. - regionReq.markSent() + regionReq.MarkSent() if err := doSend(s.createRegionRequest(region)); err != nil { state.markStopped(err) return err @@ -479,7 +479,7 @@ func (s *regionRequestWorker) processRegionSendTask( if err := drainControl(); err != nil { return err } - if regionReq = s.requestCache.tryPop(); regionReq != nil { + if regionReq = s.requestCache.TryPop(); regionReq != nil { continue } select { @@ -520,7 +520,7 @@ func (s *regionRequestWorker) addRegionState(subscriptionID SubscriptionID, regi zap.Uint64("subscriptionID", uint64(subscriptionID)), zap.Uint64("regionID", regionID)) if oldState.request != nil { - oldState.request.finish() + oldState.request.Finish() } } states[regionID] = state @@ -565,10 +565,12 @@ func (s *regionRequestWorker) clearRegionStates() map[SubscriptionID]regionFeedS return subscriptions } -// add adds a region request to the worker's cache +// Add adds a region request to the worker's cache. // It blocks if the cache is full until there's space or ctx is cancelled -func (s *regionRequestWorker) add(ctx context.Context, region regionInfo, force bool) (bool, error) { - return s.requestCache.add(ctx, region, force) +func (s *regionRequestWorker) Add( + ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, +) (bool, error) { + return s.requestCache.Add(ctx, region, force, quota) } func (s *regionRequestWorker) clearPendingRegions() []regionInfo { @@ -579,13 +581,13 @@ func (s *regionRequestWorker) clearPendingRegions() []regionInfo { req := s.preFetchForConnecting s.preFetchForConnecting = nil regions = append(regions, req.regionInfo) - req.finish() + req.Finish() } - regions = append(regions, s.requestCache.takeUnsentRegions()...) + regions = append(regions, s.requestCache.TakeUnsentRegions()...) return regions } -func (s *regionRequestWorker) releaseAdmittedRegionRequests() { - s.requestCache.clear() +func (s *regionRequestWorker) ReleaseAdmittedRegionRequests() { + s.requestCache.Clear() } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 7707441a91..8e8df894cf 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -85,25 +85,25 @@ func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req1, err := worker.requestCache.pop(ctx) + req1, err := worker.requestCache.Pop(ctx) require.NoError(t, err) - req1.markSent() + req1.MarkSent() state1 := newRegionFeedState(req1.regionInfo, uint64(req1.regionInfo.subscribedSpan.subID), worker, req1) worker.addRegionState(req1.regionInfo.subscribedSpan.subID, req1.regionInfo.verID.GetID(), state1) - ok, err = worker.requestCache.add(ctx, region, false) + ok, err = worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 2, worker.requestCache.getPendingCount()) - req2, err := worker.requestCache.pop(ctx) + require.Equal(t, 2, worker.requestCache.PendingCount()) + req2, err := worker.requestCache.Pop(ctx) require.NoError(t, err) state2 := newRegionFeedState(req2.regionInfo, uint64(req2.regionInfo.subscribedSpan.subID), worker, req2) worker.addRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID(), state2) - require.Equal(t, 1, worker.requestCache.getPendingCount()) + require.Equal(t, 1, worker.requestCache.PendingCount()) require.Same(t, state2, worker.getRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID())) } @@ -115,20 +115,20 @@ func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.Pop(ctx) require.NoError(t, err) - require.Equal(t, 1, worker.requestCache.getPendingCount()) + require.Equal(t, 1, worker.requestCache.PendingCount()) worker.preFetchForConnecting = req regions := worker.clearPendingRegions() require.Len(t, regions, 1) require.Nil(t, worker.preFetchForConnecting) - require.Equal(t, 0, worker.requestCache.getPendingCount()) + require.Equal(t, 0, worker.requestCache.PendingCount()) } type pushedResolvedEvent struct { @@ -302,11 +302,11 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.Pop(ctx) require.NoError(t, err) state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) @@ -317,11 +317,11 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { // visible as sent, a fast region error may mark the region stopped before // worker cleanup runs. In that case, markStopped should finish the request // immediately, so clearPendingRegions must not return the stale region again. - req.markSent() + req.MarkSent() state.markStopped(errors.New("send request to store error")) worker.takeRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) - require.Equal(t, 0, worker.requestCache.getPendingCount()) + require.Equal(t, 0, worker.requestCache.PendingCount()) require.Empty(t, worker.clearPendingRegions()) } @@ -339,12 +339,12 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 1, worker.requestCache.getPendingCount()) + require.Equal(t, 1, worker.requestCache.PendingCount()) - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.Pop(ctx) require.NoError(t, err) worker.preFetchForConnecting = req @@ -356,7 +356,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { err = worker.processRegionSendTask(ctx, conn) require.ErrorIs(t, err, sendErr) - require.Equal(t, 0, worker.requestCache.getPendingCount()) + require.Equal(t, 0, worker.requestCache.PendingCount()) state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") } @@ -391,11 +391,11 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.add(ctx, region, false) + ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.pop(ctx) + req, err := worker.requestCache.Pop(ctx) require.NoError(t, err) worker.preFetchForConnecting = req @@ -407,7 +407,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { err = worker.processRegionSendTask(ctx, conn) var streamErr *storeStreamErr require.ErrorAs(t, err, &streamErr) - require.Equal(t, 0, worker.requestCache.getPendingCount()) + require.Equal(t, 0, worker.requestCache.PendingCount()) state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index b73b36a05a..909b48e76e 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -132,7 +132,7 @@ func (s *regionFeedState) markStopped(err error) { s.state.err = err } if s.request != nil { - s.request.finish() + s.request.Finish() } } @@ -146,7 +146,7 @@ func (s *regionFeedState) markRemoved() (changed bool) { s.matcher.clear() } if s.request != nil { - s.request.finish() + s.request.Finish() } return } @@ -172,7 +172,7 @@ func (s *regionFeedState) isInitialized() bool { func (s *regionFeedState) setInitialized() { s.region.lockedRangeState.Initialized.Store(true) if s.request != nil { - s.request.resolve() + s.request.Resolve() } } diff --git a/logservice/logpuller/subscription_client.go b/logservice/logpuller/subscription_client.go index ec263c7ccf..a5a0ee2b8f 100644 --- a/logservice/logpuller/subscription_client.go +++ b/logservice/logpuller/subscription_client.go @@ -82,6 +82,8 @@ type rangeTask struct { type SubscriptionClientConfig struct { // The number of region request workers to send region task for every tikv store RegionRequestWorkerPerStore uint + // PendingRegionRequestQueueSize is the total pending region request quota for one TiKV store. + PendingRegionRequestQueueSize int } type upstreamHandle struct { @@ -92,7 +94,7 @@ type upstreamHandle struct { clusterID uint64 } -func (u *upstreamHandle) initialize(ctx context.Context) { +func (u *upstreamHandle) Initialize(ctx context.Context) { u.clusterID = u.pd.GetClusterID(ctx) } @@ -215,7 +217,7 @@ func (s *subscriptionClient) runMetricsUpdater(ctx context.Context) error { ).Set(float64(areaMetric.MemoryUsage())) } - s.regionScheduler.updateMetrics() + s.regionScheduler.UpdateMetrics() s.spanRegistry.UpdateMetrics() } } @@ -283,7 +285,7 @@ func (s *subscriptionClient) Unsubscribe(subID SubscriptionID) { } func (s *subscriptionClient) Run(ctx context.Context) error { - s.upstream.initialize(ctx) + s.upstream.Initialize(ctx) g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return s.eventSink.Run(ctx) }) @@ -304,7 +306,7 @@ func (s *subscriptionClient) Close(ctx context.Context) error { s.cancel() s.eventSink.Close() if s.regionScheduler != nil { - s.regionScheduler.close() + s.regionScheduler.Close() } return nil } @@ -316,7 +318,7 @@ func (s *subscriptionClient) setTableStopped(rt *subscribedSpan) { // Set stopped to true so we can stop handling region events from the table. // Then broadcast deregister requests to all region request workers. if rt.stopped.CompareAndSwap(false, true) { - s.regionScheduler.broadcastDeregister(rt.subID, rt.filterLoop) + s.regionScheduler.BroadcastDeregister(rt.subID, rt.filterLoop) if rt.rangeLock.Stop() { s.onTableDrained(rt) } @@ -451,7 +453,7 @@ func (s *subscriptionClient) scheduleRegionRequest(ctx context.Context, region r switch lockRangeResult.Status { case regionlock.LockRangeStatusSuccess: region.lockedRangeState = lockRangeResult.LockedRangeState - s.regionScheduler.submit(priority, region) + s.regionScheduler.Submit(priority, region) case regionlock.LockRangeStatusStale: for _, r := range lockRangeResult.RetryRanges { s.scheduleRangeRequest(ctx, r, region.subscribedSpan, region.filterLoop, priority) diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index 92fa18f7b3..b557b100f9 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/ticdc/pkg/metrics" "github.com/pingcap/ticdc/pkg/pdutil" "github.com/pingcap/ticdc/pkg/security" + "github.com/pingcap/ticdc/utils/chann" "github.com/pingcap/ticdc/utils/dynstream" "github.com/pingcap/ticdc/utils/priorityqueue" "github.com/pingcap/tidb/pkg/store/mockstore/mockcopr" @@ -389,6 +390,39 @@ func TestOnRegionFailQueuesCanceledErrorCache(t *testing.T) { require.Nil(t, client.spanRegistry.Get(span.subID)) } +func TestRegionFailureHandlerReschedulesRegionWhenRPCCtxChanged(t *testing.T) { + var regionScheduleCount int + var rangeScheduleCount int + var scheduledPriority TaskType + handler := newRegionFailureHandler( + &upstreamHandle{}, + func(*subscribedSpan) {}, + func(_ context.Context, _ regionInfo, priority TaskType) { + regionScheduleCount++ + scheduledPriority = priority + }, + func(_ context.Context, _ heartbeatpb.TableSpan, _ *subscribedSpan, _ bool, _ TaskType) { + rangeScheduleCount++ + }, + ) + + span := &subscribedSpan{subID: SubscriptionID(1)} + region := regionInfo{ + verID: tikv.NewRegionVerID(1, 1, 1), + subscribedSpan: span, + } + err := handler.handleError(t.Context(), newRegionErrorInfo(region, &rpcCtxChangedError{ + verID: region.verID, + from: "store-1", + to: "store-2", + })) + + require.NoError(t, err) + require.Equal(t, 1, regionScheduleCount) + require.Equal(t, 0, rangeScheduleCount) + require.Equal(t, TaskHighPrior, scheduledPriority) +} + type mockDynamicStream struct{} func (s *mockDynamicStream) Start() {} @@ -465,49 +499,74 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - ok, err := worker.add(ctx, dummyRegion, true) + ok, err := worker.Add(ctx, dummyRegion, true, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - scheduler.broadcastDeregister(SubscriptionID(1), true) + scheduler.BroadcastDeregister(SubscriptionID(1), true) require.Equal(t, 1, worker.controlQueue.ch.Len()) req, ok := worker.controlQueue.tryPop() require.True(t, ok) require.Equal(t, SubscriptionID(1), req.subID) require.True(t, req.filterLoop) - require.Equal(t, 1, worker.requestCache.getPendingCount()) + require.Equal(t, 1, worker.requestCache.PendingCount()) } -func TestRequestedStoreDeferredTasksBlockSameStore(t *testing.T) { - queue := priorityqueue.New[*regionPriorityTask]() - scheduler := ®ionRequestScheduler{queue: queue} - store := &requestedStore{scheduler: scheduler, storeAddr: "store-1"} +func TestRequestedStoreDeferredTasksPriority(t *testing.T) { + store := &requestedStore{storeAddr: "store-1", pendingTasks: priorityqueue.New[*regionPriorityTask]()} currentTs := oracle.GoTimeToTS(time.Now()) span := &subscribedSpan{subID: SubscriptionID(1)} span.resolvedTs.Store(currentTs) region := regionInfo{subscribedSpan: span} - firstTask := newRegionPriorityTask(TaskHighPrior, region, currentTs, 1) - secondTask := newRegionPriorityTask(TaskHighPrior, region, currentTs, 2) + lowTask := newRegionPriorityTask(TaskLowPrior, region, currentTs, 1) + highTask := newRegionPriorityTask(TaskHighPrior, region, currentTs, 2) - store.deferTask(firstTask) - require.True(t, store.hasDeferredTaskAhead(secondTask)) + store.pendingTasks.Push(lowTask) + store.pendingTasks.Push(highTask) - store.promoteDeferredTask() - promoted, err := queue.Pop(t.Context()) - require.NoError(t, err) - require.Same(t, firstTask, promoted) - require.False(t, store.hasDeferredTaskAhead(firstTask)) - require.True(t, store.hasDeferredTaskAhead(secondTask)) + task, ok := store.pendingTasks.TryPop() + require.True(t, ok) + require.Same(t, highTask, task) + task, ok = store.pendingTasks.TryPop() + require.True(t, ok) + require.Same(t, lowTask, task) + _, ok = store.pendingTasks.TryPop() + require.False(t, ok) +} + +func TestRequestedStoreQuotaReleaseNotifiesSchedulerOnce(t *testing.T) { + scheduler := ®ionRequestScheduler{ + schedulerNotify: make(chan struct{}, 1), + storeAvailable: chann.NewUnlimitedChannelDefault[*requestedStore](), + } + store := &requestedStore{scheduler: scheduler, storeAddr: "store-1"} + store.quota = newStoreQuota(1, store.NotifyAvailable) + + quota, ok := store.quota.TryAcquire() + require.True(t, ok) + _, ok = store.quota.TryAcquire() + require.False(t, ok) + + quota.Release() + quota.Release() + require.Equal(t, 1, scheduler.storeAvailable.Len()) + select { + case <-scheduler.schedulerNotify: + default: + require.Fail(t, "quota release should notify scheduler") + } - store.deferTask(firstTask) - store.promoteDeferredTask() - promoted, err = queue.Pop(t.Context()) + readyStore, ok, err := scheduler.storeAvailable.GetWithContext(context.Background()) require.NoError(t, err) - require.Same(t, firstTask, promoted) + require.True(t, ok) + require.Same(t, store, readyStore) + readyStore.MarkAvailableDequeued() - store.finishPromotedTask(firstTask) - require.False(t, store.hasDeferredTaskAhead(secondTask)) + quota, ok = store.quota.TryAcquire() + require.True(t, ok) + quota.Release() + require.Equal(t, 1, scheduler.storeAvailable.Len()) } func TestSubscriptionWithFailedTiKV(t *testing.T) { @@ -540,7 +599,8 @@ func TestSubscriptionWithFailedTiKV(t *testing.T) { cluster.Bootstrap(11, []uint64{1, 2, 3}, []uint64{4, 5, 6}, 6) clientConfig := &SubscriptionClientConfig{ - RegionRequestWorkerPerStore: 2, + RegionRequestWorkerPerStore: 2, + PendingRegionRequestQueueSize: 32, } client := NewSubscriptionClient( clientConfig, diff --git a/server/server.go b/server/server.go index b9dc5edad8..2e83359930 100644 --- a/server/server.go +++ b/server/server.go @@ -199,7 +199,8 @@ func (c *server) initialize(ctx context.Context) error { schemaStore := schemastore.New(conf.DataDir, c.pdClient) subscriptionClient := logpuller.NewSubscriptionClient( &logpuller.SubscriptionClientConfig{ - RegionRequestWorkerPerStore: 8, + RegionRequestWorkerPerStore: 8, + PendingRegionRequestQueueSize: conf.Debug.Puller.PendingRegionRequestQueueSize, }, c.pdClient, txnutil.NewLockerResolver(), c.security, From ea50ad01f9313ec4fe20c4a9be1cbf10b099c0c0 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Mon, 29 Jun 2026 15:05:50 -0700 Subject: [PATCH 18/21] refactor --- logservice/logpuller/region_request_cache.go | 134 +++++--------- .../logpuller/region_request_cache_test.go | 78 ++++---- .../logpuller/region_request_scheduler.go | 148 +-------------- logservice/logpuller/region_request_worker.go | 161 ++++++++--------- .../logpuller/region_request_worker_test.go | 51 +++--- logservice/logpuller/requested_store.go | 171 ++++++++++++++++++ .../logpuller/subscription_client_test.go | 4 +- 7 files changed, 362 insertions(+), 385 deletions(-) create mode 100644 logservice/logpuller/requested_store.go diff --git a/logservice/logpuller/region_request_cache.go b/logservice/logpuller/region_request_cache.go index 1bef819570..6d77409fcc 100644 --- a/logservice/logpuller/region_request_cache.go +++ b/logservice/logpuller/region_request_cache.go @@ -14,6 +14,7 @@ package logpuller import ( + "container/list" "context" "sync" "time" @@ -41,8 +42,6 @@ const ( // regionReqStageSent means the request has been sent to TiKV and is waiting // for initialized/resolved/stopped cleanup. regionReqStageSent - // regionReqStageFinished means the request has left the worker window. - regionReqStageFinished ) // regionReq tracks one data request from admission to completion. @@ -54,9 +53,11 @@ type regionReq struct { // when the request leaves the worker window. quota *regionRequestQuota - // cache is set by requestCache.Add. It lets regionFeedState finish exactly + // cache is set by requestCache.add. It lets regionFeedState finish exactly // the request that created it instead of looking up by subID/regionID. cache *requestCache + // readyElement is set while the request is waiting in requestCache.ready. + readyElement *list.Element // stage is guarded by requestCache.mu. stage regionReqStage } @@ -90,29 +91,26 @@ func (r *regionReq) ReleaseQuota() { // requestCache manages worker-local data requests with flow control. // // requests is the source of truth for live requests. A request is inserted by -// Add(), moves through queued/processing/sent, and is removed by resolve(), -// finish(), TakeUnsentRegions(), or Clear(). +// add(), moves through queued/processing/sent, and is removed by resolve(), +// finish(), takeUnsentRegions(), or clear(). type requestCache struct { mu sync.Mutex // requests owns every live data request in this worker. Its length is the - // flow-control count used by Add(), PendingCount(), and metrics. + // flow-control count used by add(), pendingCount(), and metrics. requests map[*regionReq]struct{} - // ready is the FIFO list of queued requests. Entries already popped or - // removed are left as nil/stale and skipped by TryPop; readyIdx is the next - // candidate index. compactReadyLocked occasionally drops skipped entries. - ready []*regionReq - readyIdx int + // ready is the FIFO list of queued requests. + ready *list.List // maxPendingCount limits len(requests) for non-force adds. force adds bypass // this limit, matching the old force behavior for high-priority data requests. maxPendingCount int - // readyAvailable wakes a worker blocked in Pop() when a queued request is + // readyAvailable wakes a worker blocked in pop() when a queued request is // appended. It is a level-trigger hint; callers must re-check ready under mu. readyAvailable chan struct{} - // spaceAvailable wakes Add() when a live request leaves requests. + // spaceAvailable wakes add() when a live request leaves requests. spaceAvailable chan struct{} // onSpaceAvailable lets the store-level deferred scheduler retry a task when // this worker frees request capacity. @@ -122,7 +120,7 @@ type requestCache struct { func newRequestCache(maxPendingCount int, onSpaceAvailable func()) *requestCache { return &requestCache{ requests: make(map[*regionReq]struct{}), - ready: make([]*regionReq, 0, maxPendingCount), + ready: list.New(), maxPendingCount: maxPendingCount, readyAvailable: make(chan struct{}, 1), spaceAvailable: make(chan struct{}, 1), @@ -130,8 +128,8 @@ func newRequestCache(maxPendingCount int, onSpaceAvailable func()) *requestCache } } -// Add admits a data request into the worker window. -func (c *requestCache) Add( +// add admits a data request into the worker window. +func (c *requestCache) add( ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, ) (bool, error) { start := time.Now() @@ -175,15 +173,15 @@ func (c *requestCache) tryAdd(region regionInfo, force bool, quota *regionReques req := newRegionReq(c, region, quota) c.requests[req] = struct{}{} - c.ready = append(c.ready, req) + req.readyElement = c.ready.PushBack(req) notifyReady = true return true } // pop takes the next queued request and moves it into processing state. -func (c *requestCache) Pop(ctx context.Context) (*regionReq, error) { +func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { for { - if req := c.TryPop(); req != nil { + if req := c.tryPop(); req != nil { return req, nil } @@ -195,29 +193,19 @@ func (c *requestCache) Pop(ctx context.Context) (*regionReq, error) { } } -func (c *requestCache) TryPop() *regionReq { +func (c *requestCache) tryPop() *regionReq { c.mu.Lock() defer c.mu.Unlock() - for c.readyIdx < len(c.ready) { - req := c.ready[c.readyIdx] - c.ready[c.readyIdx] = nil - c.readyIdx++ - - if req == nil { - continue - } - if _, ok := c.requests[req]; !ok || req.stage != regionReqStageQueued { - continue - } - - req.stage = regionReqStageProcessing - c.compactReadyLocked() - return req + elem := c.ready.Front() + if elem == nil { + return nil } - - c.compactReadyLocked() - return nil + req := elem.Value.(*regionReq) + c.ready.Remove(elem) + req.readyElement = nil + req.stage = regionReqStageProcessing + return req } func (c *requestCache) markSent(req *regionReq) { @@ -239,7 +227,7 @@ func (c *requestCache) resolve(req *regionReq) bool { zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), zap.Uint64("regionID", req.regionInfo.verID.GetID()), zap.Float64("cost", cost), - zap.Int("pendingCount", c.PendingCount())) + zap.Int("pendingCount", c.pendingCount())) metrics.RegionRequestFinishScanDuration.Observe(cost) return true } @@ -269,62 +257,53 @@ func (c *requestCache) remove(req *regionReq) bool { return removed } -func (c *requestCache) TakeUnsentRegions() []regionInfo { +func (c *requestCache) takeUnsentRegions() []regionInfo { c.mu.Lock() - regions := make([]regionInfo, 0, len(c.requests)) removedReqs := make([]*regionReq, 0, len(c.requests)) - removed := 0 + regions := make([]regionInfo, 0, len(c.requests)) for req := range c.requests { if req.stage == regionReqStageSent { continue } - regions = append(regions, req.regionInfo) if c.removeLocked(req) { - removed++ removedReqs = append(removedReqs, req) + regions = append(regions, req.regionInfo) } } - if removed > 0 { - c.compactReadyLocked() - } c.mu.Unlock() - if removed > 0 { - for _, req := range removedReqs { - req.ReleaseQuota() - } - c.notifySpace() - } + c.releaseRemovedReqs(removedReqs) return regions } // clear removes all live requests and returns their regions. -func (c *requestCache) Clear() []regionInfo { +func (c *requestCache) clear() []regionInfo { c.mu.Lock() - regions := make([]regionInfo, 0, len(c.requests)) removedReqs := make([]*regionReq, 0, len(c.requests)) + regions := make([]regionInfo, 0, len(c.requests)) for req := range c.requests { - regions = append(regions, req.regionInfo) - removedReqs = append(removedReqs, req) - delete(c.requests, req) - req.stage = regionReqStageFinished + if c.removeLocked(req) { + removedReqs = append(removedReqs, req) + regions = append(regions, req.regionInfo) + } } - removed := len(regions) - c.ready = c.ready[:0] - c.readyIdx = 0 c.mu.Unlock() - if removed > 0 { + c.releaseRemovedReqs(removedReqs) + return regions +} + +func (c *requestCache) releaseRemovedReqs(removedReqs []*regionReq) { + if len(removedReqs) > 0 { for _, req := range removedReqs { req.ReleaseQuota() } c.notifySpace() } - return regions } -// getPendingCount returns the number of queued, processing and sent requests. -func (c *requestCache) PendingCount() int { +// pendingCount returns the number of queued, processing and sent requests. +func (c *requestCache) pendingCount() int { c.mu.Lock() defer c.mu.Unlock() return len(c.requests) @@ -338,11 +317,10 @@ func (c *requestCache) removeLocked(req *regionReq) bool { return false } - stage := req.stage delete(c.requests, req) - req.stage = regionReqStageFinished - if stage != regionReqStageSent { - c.compactReadyLocked() + if req.readyElement != nil { + c.ready.Remove(req.readyElement) + req.readyElement = nil } return true } @@ -363,19 +341,3 @@ func (c *requestCache) notifySpace() { c.onSpaceAvailable() } } - -func (c *requestCache) compactReadyLocked() { - if c.readyIdx == 0 { - return - } - if c.readyIdx < len(c.ready) && c.readyIdx < 1024 { - return - } - - n := copy(c.ready, c.ready[c.readyIdx:]) - for i := n; i < len(c.ready); i++ { - c.ready[i] = nil - } - c.ready = c.ready[:n] - c.readyIdx = 0 -} diff --git a/logservice/logpuller/region_request_cache_test.go b/logservice/logpuller/region_request_cache_test.go index e353cbee13..fbe8614a17 100644 --- a/logservice/logpuller/region_request_cache_test.go +++ b/logservice/logpuller/region_request_cache_test.go @@ -52,13 +52,13 @@ func TestRequestCacheAdd_NormalCase(t *testing.T) { region := createTestRegionInfo(1, 1) - ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) // Verify the request was added to the queue - req, err := cache.Pop(ctx) + req, err := cache.pop(ctx) require.NoError(t, err) require.NotNil(t, req) require.Equal(t, region.verID.GetID(), req.regionInfo.verID.GetID()) @@ -71,35 +71,35 @@ func TestRequestCacheAdd_ForceFlag(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) - ok, err := cache.Add(ctx, region1, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) // Try to add another request without force - should fail due to retry limit region2 := createTestRegionInfo(1, 2) - ok, err = cache.Add(ctx, region2, false, testRegionRequestQuota()) + ok, err = cache.add(ctx, region2, false, testRegionRequestQuota()) require.False(t, ok) require.NoError(t, err) // With force=true, the request bypasses the live request limit. region3 := createTestRegionInfo(1, 3) - ok, err = cache.Add(ctx, region3, true, testRegionRequestQuota()) + ok, err = cache.add(ctx, region3, true, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.PendingCount()) + require.Equal(t, 2, cache.pendingCount()) - req, err := cache.Pop(ctx) + req, err := cache.pop(ctx) require.NoError(t, err) require.NotNil(t, req) require.Equal(t, region1.verID.GetID(), req.regionInfo.verID.GetID()) require.Equal(t, region1.subscribedSpan.subID, req.regionInfo.subscribedSpan.subID) - require.Equal(t, 2, cache.PendingCount()) + require.Equal(t, 2, cache.pendingCount()) req.MarkSent() // resolve region1 req.Resolve() - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) } func TestRequestCacheAdd_ContextCancellation(t *testing.T) { @@ -108,7 +108,7 @@ func TestRequestCacheAdd_ContextCancellation(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) ctx1 := context.Background() - ok, err := cache.Add(ctx1, region1, false, testRegionRequestQuota()) + ok, err := cache.add(ctx1, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) @@ -117,7 +117,7 @@ func TestRequestCacheAdd_ContextCancellation(t *testing.T) { cancel() // Cancel immediately region2 := createTestRegionInfo(1, 2) - ok, err = cache.Add(ctx2, region2, false, testRegionRequestQuota()) + ok, err = cache.add(ctx2, region2, false, testRegionRequestQuota()) require.False(t, ok) require.Error(t, err) require.Equal(t, context.Canceled, err) @@ -129,13 +129,13 @@ func TestRequestCacheAdd_RetryLimitExceeded(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) - ok, err := cache.Add(ctx, region1, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) // Try to add another request - should eventually hit retry limit region2 := createTestRegionInfo(1, 2) - ok, err = cache.Add(ctx, region2, false, testRegionRequestQuota()) + ok, err = cache.add(ctx, region2, false, testRegionRequestQuota()) require.False(t, ok) require.NoError(t, err) } @@ -146,36 +146,36 @@ func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { // Fill up the cache region1 := createTestRegionInfo(1, 1) - ok, err := cache.Add(ctx, region1, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region1, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) region2 := createTestRegionInfo(1, 2) - ok, err = cache.Add(ctx, region2, false, testRegionRequestQuota()) + ok, err = cache.add(ctx, region2, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.PendingCount()) + require.Equal(t, 2, cache.pendingCount()) // Pop a request and mark it as sent, then resolve it to free up space - req, err := cache.Pop(ctx) + req, err := cache.pop(ctx) require.NoError(t, err) require.NotNil(t, req) - require.Equal(t, 2, cache.PendingCount()) // pop doesn't change pendingCount + require.Equal(t, 2, cache.pendingCount()) // pop doesn't change pendingCount req.MarkSent() - require.Equal(t, 2, cache.PendingCount()) + require.Equal(t, 2, cache.pendingCount()) // Resolve the request to free up space success := req.Resolve() require.True(t, success) - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) // Now we should be able to add another request region3 := createTestRegionInfo(1, 3) - ok, err = cache.Add(ctx, region3, false, testRegionRequestQuota()) + ok, err = cache.add(ctx, region3, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.PendingCount()) + require.Equal(t, 2, cache.pendingCount()) } func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { @@ -189,7 +189,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { for i := range numGoroutines { go func(id int) { region := createTestRegionInfo(SubscriptionID(id%3), uint64(id)) - ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) done <- err @@ -206,7 +206,7 @@ func TestRequestCacheAdd_ConcurrentAdds(t *testing.T) { } } - require.Equal(t, numGoroutines, cache.PendingCount()) + require.Equal(t, numGoroutines, cache.pendingCount()) } func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testing.T) { @@ -215,18 +215,18 @@ func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testi region := createTestRegionInfo(1, 1) - ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - ok, err = cache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err = cache.add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 2, cache.PendingCount()) + require.Equal(t, 2, cache.pendingCount()) - req1, err := cache.Pop(ctx) + req1, err := cache.pop(ctx) require.NoError(t, err) - req2, err := cache.Pop(ctx) + req2, err := cache.pop(ctx) require.NoError(t, err) require.NotSame(t, req1, req2) } @@ -237,19 +237,19 @@ func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { region := createTestRegionInfo(1, 1) - ok, err := cache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := cache.add(ctx, region, false, testRegionRequestQuota()) require.True(t, ok) require.NoError(t, err) - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) - req, err := cache.Pop(ctx) + req, err := cache.pop(ctx) require.NoError(t, err) req.MarkSent() - require.Equal(t, 1, cache.PendingCount()) + require.Equal(t, 1, cache.pendingCount()) req.Finish() - require.Equal(t, 0, cache.PendingCount()) + require.Equal(t, 0, cache.pendingCount()) } func TestRequestCacheFinishReleasesQuotaOnce(t *testing.T) { @@ -263,11 +263,11 @@ func TestRequestCacheFinishReleasesQuotaOnce(t *testing.T) { }, } - ok, err := cache.Add(ctx, region, false, quota) + ok, err := cache.add(ctx, region, false, quota) require.True(t, ok) require.NoError(t, err) - req, err := cache.Pop(ctx) + req, err := cache.pop(ctx) require.NoError(t, err) require.True(t, req.Finish()) require.False(t, req.Finish()) diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index d69fea692c..66212ccf2d 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -144,7 +144,7 @@ func (s *regionRequestScheduler) UpdateMetrics() { s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) for _, worker := range store.requestWorkers { - count += worker.requestCache.PendingCount() + count += worker.PendingRequestCount() } return true }) @@ -281,149 +281,3 @@ func (s *regionRequestScheduler) tryAdmitTask( zap.String("addr", store.storeAddr)) return true, nil } - -type regionRequestQuota struct { - once sync.Once - release func() -} - -func (p *regionRequestQuota) Release() { - p.once.Do(p.release) -} - -type storeQuota struct { - mu sync.Mutex - capacity int - used int - - onRelease func() -} - -func newStoreQuota(capacity int, onRelease func()) *storeQuota { - return &storeQuota{ - capacity: capacity, - onRelease: onRelease, - } -} - -func (q *storeQuota) TryAcquire() (*regionRequestQuota, bool) { - q.mu.Lock() - if q.used >= q.capacity { - q.mu.Unlock() - return nil, false - } - q.used++ - q.mu.Unlock() - - return ®ionRequestQuota{ - release: func() { - q.mu.Lock() - q.used-- - q.mu.Unlock() - if q.onRelease != nil { - q.onRelease() - } - }, - }, true -} - -// requestedStore is the scheduler's local state for one TiKV store. -type requestedStore struct { - scheduler *regionRequestScheduler - - storeAddr string - quota *storeQuota - // nextWorker is the round-robin cursor used to select the next worker to attempt. - nextWorker atomic.Uint32 - // requestWorkers are fully created before requestedStore is published - // and remain immutable afterwards. - requestWorkers []*regionRequestWorker - - // pendingTasks holds tasks that have been routed to this store but are - // waiting for store quota or worker request-cache capacity. It is mutated - // only by the single regionRequestScheduler.Run loop. - pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] - - notifyMu sync.Mutex - notifyEnqueued bool -} - -func newRequestedStore( - ctx context.Context, - eg *errgroup.Group, - scheduler *regionRequestScheduler, - storeAddr string, -) *requestedStore { - pendingRegionRequestQueueSize := scheduler.config.PendingRegionRequestQueueSize - regionRequestWorkerPerStore := scheduler.config.RegionRequestWorkerPerStore - perWorkerQueueSize := pendingRegionRequestQueueSize / int(regionRequestWorkerPerStore) - if perWorkerQueueSize <= 0 { - log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", - zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), - zap.Uint("regionRequestWorkerPerStore", regionRequestWorkerPerStore)) - perWorkerQueueSize = 1 - } - perStoreQuotaSize := perWorkerQueueSize * int(regionRequestWorkerPerStore) - - rs := &requestedStore{ - scheduler: scheduler, - storeAddr: storeAddr, - requestWorkers: make([]*regionRequestWorker, 0, regionRequestWorkerPerStore), - pendingTasks: priorityqueue.New[*regionPriorityTask](), - } - rs.quota = newStoreQuota(perStoreQuotaSize, rs.NotifyAvailable) - for range regionRequestWorkerPerStore { - requestWorker := newRegionRequestWorker( - ctx, - eg, - rs, - perWorkerQueueSize, - scheduler.upstream, - scheduler.eventSink, - scheduler.failureHandler, - ) - rs.requestWorkers = append(rs.requestWorkers, requestWorker) - } - return rs -} - -func (rs *requestedStore) Close() { - for _, worker := range rs.requestWorkers { - worker.ReleaseAdmittedRegionRequests() - } -} - -func (rs *requestedStore) AddRegion( - ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, -) (bool, *regionRequestWorker, error) { - workers := rs.requestWorkers - - start := int(rs.nextWorker.Add(1)) % len(workers) - for i := range len(workers) { - worker := workers[(start+i)%len(workers)] - ok, err := worker.Add(ctx, region, force, quota) - if err != nil || ok { - return ok, worker, err - } - } - return false, nil, nil -} - -func (rs *requestedStore) NotifyAvailable() { - rs.notifyMu.Lock() - if rs.notifyEnqueued { - rs.notifyMu.Unlock() - return - } - rs.notifyEnqueued = true - rs.notifyMu.Unlock() - - rs.scheduler.storeAvailable.Push(rs) - rs.scheduler.notifyScheduler() -} - -func (rs *requestedStore) MarkAvailableDequeued() { - rs.notifyMu.Lock() - rs.notifyEnqueued = false - rs.notifyMu.Unlock() -} diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 5a0e887dbf..272d63cc7d 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -86,10 +86,6 @@ type regionRequestWorker struct { eventSink *regionEventSink failureHandler *regionFailureHandler - // we must always get a region to request before create a grpc stream. - // only in this way we can avoid to try to connect to an offline store infinitely. - preFetchForConnecting *regionReq - // request cache with flow control requestCache *requestCache controlQueue *controlQueue @@ -124,58 +120,23 @@ func newRegionRequestWorker( } worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - waitForPreFetching := func() error { - if worker.preFetchForConnecting != nil { - log.Panic("preFetchForConnecting should be nil", - zap.Uint64("workerID", worker.workerID), - zap.String("addr", store.storeAddr)) - } - req, err := worker.requestCache.Pop(ctx) - if err != nil { - return err - } - worker.preFetchForConnecting = req - return nil - } - g.Go(func() error { for { - if err := waitForPreFetching(); err != nil { + firstReq, err := worker.waitForRegion(ctx) + if err != nil { return err } - var regionErr error - if err := version.CheckStoreVersion(ctx, worker.upstream.pd); err != nil { - if errors.Cause(err) == context.Canceled { - return nil - } - log.Error("event feed check store version fails", - zap.Uint64("workerID", worker.workerID), - zap.String("addr", worker.store.storeAddr), - zap.Error(err)) - if cerror.Is(err, cerror.ErrGetAllStoresFailed) { - regionErr = &getStoreErr{} - } else { - regionErr = &storeStreamErr{} - } - } else { - if canceled := worker.Run(ctx, worker.upstream.credential); canceled { - return nil - } - regionErr = &storeStreamErr{} + err = worker.checkStoreVersion(ctx) + if err == nil { + err = worker.Run(ctx, worker.upstream.credential, firstReq) } - for subID, m := range worker.clearRegionStates() { - for _, state := range m { - state.markStopped(regionErr) - regionEvent := regionEvent{ - states: []*regionFeedState{state}, - } - worker.eventSink.Push(subID, regionEvent) - } + if ctx.Err() != nil { + return ctx.Err() } - // The store may fail forever, so we need try to re-schedule all pending regions. - for _, region := range worker.clearPendingRegions() { - worker.failureHandler.Report(newRegionErrorInfo(region, regionErr)) + if err == nil { + err = &storeStreamErr{} } + worker.handleStoreFailure(err) if err := util.Hang(ctx, time.Second); err != nil { return err } @@ -185,16 +146,51 @@ func newRegionRequestWorker( return worker } -func (s *regionRequestWorker) Run(ctx context.Context, credential *security.Credential) (canceled bool) { - isCanceled := func() bool { - select { - case <-ctx.Done(): - return true - default: - return false +func (s *regionRequestWorker) waitForRegion(ctx context.Context) (*regionReq, error) { + req, err := s.requestCache.pop(ctx) + if err != nil { + return nil, err + } + return req, nil +} + +func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { + err := version.CheckStoreVersion(ctx, s.upstream.pd) + if err == nil { + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + log.Error("event feed check store version fails", + zap.Uint64("workerID", s.workerID), + zap.String("addr", s.store.storeAddr), + zap.Error(err)) + if cerror.Is(err, cerror.ErrGetAllStoresFailed) { + return &getStoreErr{} + } + return &storeStreamErr{} +} + +func (s *regionRequestWorker) handleStoreFailure(regionErr error) { + for subID, m := range s.clearRegionStates() { + for _, state := range m { + state.markStopped(regionErr) + regionEvent := regionEvent{ + states: []*regionFeedState{state}, + } + s.eventSink.Push(subID, regionEvent) } } + // The store may fail forever, so we need try to re-schedule all pending regions. + for _, region := range s.clearPendingRegions() { + s.failureHandler.Report(newRegionErrorInfo(region, regionErr)) + } +} +func (s *regionRequestWorker) Run( + ctx context.Context, credential *security.Credential, firstReq *regionReq, +) (err error) { log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr)) @@ -203,7 +199,7 @@ func (s *regionRequestWorker) Run(ctx context.Context, credential *security.Cred log.Info("region request worker exits", zap.Uint64("workerID", s.workerID), zap.String("addr", s.store.storeAddr), - zap.Bool("canceled", canceled)) + zap.Error(err)) }() g, gctx := errgroup.WithContext(ctx) @@ -217,7 +213,10 @@ func (s *regionRequestWorker) Run(ctx context.Context, credential *security.Cred if conn != nil && conn.Conn != nil { _ = conn.Conn.Close() } - return isCanceled() + if ctx.Err() != nil { + return ctx.Err() + } + return &storeStreamErr{} } defer func() { _ = conn.Conn.Close() @@ -226,7 +225,7 @@ func (s *regionRequestWorker) Run(ctx context.Context, credential *security.Cred g.Go(func() error { return s.receiveAndDispatchChangeEvents(conn) }) - g.Go(func() error { return s.processRegionSendTask(gctx, conn) }) + g.Go(func() error { return s.processRegionSendTask(gctx, conn, firstReq) }) failpoint.Inject("InjectForceReconnect", func() { timer := time.After(10 * time.Second) @@ -238,8 +237,14 @@ func (s *regionRequestWorker) Run(ctx context.Context, credential *security.Cred }) }) - _ = g.Wait() - return isCanceled() + err = g.Wait() + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return &storeStreamErr{} + } + return nil } func normalizeStreamError(err error) error { @@ -384,6 +389,7 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res func (s *regionRequestWorker) processRegionSendTask( ctx context.Context, conn *ConnAndClient, + firstReq *regionReq, ) error { doSend := func(req *cdcpb.ChangeDataRequest) error { if err := conn.Client.Send(req); err != nil { @@ -431,9 +437,7 @@ func (s *regionRequestWorker) processRegionSendTask( } } - // Handle pre-fetched region first - regionReq := s.preFetchForConnecting - s.preFetchForConnecting = nil + regionReq := firstReq for { if regionReq != nil { region := regionReq.regionInfo @@ -479,7 +483,7 @@ func (s *regionRequestWorker) processRegionSendTask( if err := drainControl(); err != nil { return err } - if regionReq = s.requestCache.TryPop(); regionReq != nil { + if regionReq = s.requestCache.tryPop(); regionReq != nil { continue } select { @@ -565,29 +569,20 @@ func (s *regionRequestWorker) clearRegionStates() map[SubscriptionID]regionFeedS return subscriptions } -// Add adds a region request to the worker's cache. -// It blocks if the cache is full until there's space or ctx is cancelled -func (s *regionRequestWorker) Add( +func (s *regionRequestWorker) AddRegionRequest( ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, ) (bool, error) { - return s.requestCache.Add(ctx, region, force, quota) + return s.requestCache.add(ctx, region, force, quota) } -func (s *regionRequestWorker) clearPendingRegions() []regionInfo { - var regions []regionInfo - - // Clear pre-fetched region - if s.preFetchForConnecting != nil { - req := s.preFetchForConnecting - s.preFetchForConnecting = nil - regions = append(regions, req.regionInfo) - req.Finish() - } +func (s *regionRequestWorker) ClearRegionRequests() { + s.requestCache.clear() +} - regions = append(regions, s.requestCache.TakeUnsentRegions()...) - return regions +func (s *regionRequestWorker) PendingRequestCount() int { + return s.requestCache.pendingCount() } -func (s *regionRequestWorker) ReleaseAdmittedRegionRequests() { - s.requestCache.Clear() +func (s *regionRequestWorker) clearPendingRegions() []regionInfo { + return s.requestCache.takeUnsentRegions() } diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index 8e8df894cf..b776c559fe 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -85,29 +85,29 @@ func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req1, err := worker.requestCache.Pop(ctx) + req1, err := worker.requestCache.pop(ctx) require.NoError(t, err) req1.MarkSent() state1 := newRegionFeedState(req1.regionInfo, uint64(req1.regionInfo.subscribedSpan.subID), worker, req1) worker.addRegionState(req1.regionInfo.subscribedSpan.subID, req1.regionInfo.verID.GetID(), state1) - ok, err = worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err = worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 2, worker.requestCache.PendingCount()) - req2, err := worker.requestCache.Pop(ctx) + require.Equal(t, 2, worker.requestCache.pendingCount()) + req2, err := worker.requestCache.pop(ctx) require.NoError(t, err) state2 := newRegionFeedState(req2.regionInfo, uint64(req2.regionInfo.subscribedSpan.subID), worker, req2) worker.addRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID(), state2) - require.Equal(t, 1, worker.requestCache.PendingCount()) + require.Equal(t, 1, worker.requestCache.pendingCount()) require.Same(t, state2, worker.getRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID())) } -func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { +func TestClearPendingRegionsReleaseSlotForProcessingRegion(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10, nil), } @@ -115,20 +115,17 @@ func TestClearPendingRegionsReleaseSlotForPreFetchedRegion(t *testing.T) { ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.Pop(ctx) + _, err = worker.requestCache.pop(ctx) require.NoError(t, err) - require.Equal(t, 1, worker.requestCache.PendingCount()) - - worker.preFetchForConnecting = req + require.Equal(t, 1, worker.requestCache.pendingCount()) regions := worker.clearPendingRegions() require.Len(t, regions, 1) - require.Nil(t, worker.preFetchForConnecting) - require.Equal(t, 0, worker.requestCache.PendingCount()) + require.Equal(t, 0, worker.requestCache.pendingCount()) } type pushedResolvedEvent struct { @@ -302,11 +299,11 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { ctx := context.Background() region := createTestRegionInfo(1, 1) - ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.Pop(ctx) + req, err := worker.requestCache.pop(ctx) require.NoError(t, err) state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) @@ -321,7 +318,7 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { state.markStopped(errors.New("send request to store error")) worker.takeRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) - require.Equal(t, 0, worker.requestCache.PendingCount()) + require.Equal(t, 0, worker.requestCache.pendingCount()) require.Empty(t, worker.clearPendingRegions()) } @@ -339,14 +336,13 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - require.Equal(t, 1, worker.requestCache.PendingCount()) + require.Equal(t, 1, worker.requestCache.pendingCount()) - req, err := worker.requestCache.Pop(ctx) + req, err := worker.requestCache.pop(ctx) require.NoError(t, err) - worker.preFetchForConnecting = req sendErr := errors.New("send failed") conn := &ConnAndClient{ @@ -354,9 +350,9 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { Conn: &grpc.ClientConn{}, } - err = worker.processRegionSendTask(ctx, conn) + err = worker.processRegionSendTask(ctx, conn, req) require.ErrorIs(t, err, sendErr) - require.Equal(t, 0, worker.requestCache.PendingCount()) + require.Equal(t, 0, worker.requestCache.pendingCount()) state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") } @@ -391,23 +387,22 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) - ok, err := worker.requestCache.Add(ctx, region, false, testRegionRequestQuota()) + ok, err := worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) - req, err := worker.requestCache.Pop(ctx) + req, err := worker.requestCache.pop(ctx) require.NoError(t, err) - worker.preFetchForConnecting = req conn := &ConnAndClient{ Client: &mockEventFeedV2Client{sendErr: tc.sendErr}, Conn: &grpc.ClientConn{}, } - err = worker.processRegionSendTask(ctx, conn) + err = worker.processRegionSendTask(ctx, conn, req) var streamErr *storeStreamErr require.ErrorAs(t, err, &streamErr) - require.Equal(t, 0, worker.requestCache.PendingCount()) + require.Equal(t, 0, worker.requestCache.pendingCount()) state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go new file mode 100644 index 0000000000..05664cf763 --- /dev/null +++ b/logservice/logpuller/requested_store.go @@ -0,0 +1,171 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/utils/priorityqueue" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type regionRequestQuota struct { + once sync.Once + release func() +} + +func (p *regionRequestQuota) Release() { + p.once.Do(p.release) +} + +type storeQuota struct { + mu sync.Mutex + capacity int + used int + + onRelease func() +} + +func newStoreQuota(capacity int, onRelease func()) *storeQuota { + return &storeQuota{ + capacity: capacity, + onRelease: onRelease, + } +} + +func (q *storeQuota) TryAcquire() (*regionRequestQuota, bool) { + q.mu.Lock() + if q.used >= q.capacity { + q.mu.Unlock() + return nil, false + } + q.used++ + q.mu.Unlock() + + return ®ionRequestQuota{ + release: func() { + q.mu.Lock() + q.used-- + q.mu.Unlock() + if q.onRelease != nil { + q.onRelease() + } + }, + }, true +} + +// requestedStore is the scheduler's local state for one TiKV store. +type requestedStore struct { + scheduler *regionRequestScheduler + + storeAddr string + quota *storeQuota + // nextWorker is the round-robin cursor used to select the next worker to attempt. + nextWorker atomic.Uint32 + // requestWorkers are fully created before requestedStore is published + // and remain immutable afterwards. + requestWorkers []*regionRequestWorker + + // pendingTasks holds tasks that have been routed to this store but are + // waiting for store quota or worker request-cache capacity. It is mutated + // only by the single regionRequestScheduler.Run loop. + pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] + + notifyMu sync.Mutex + notifyEnqueued bool +} + +func newRequestedStore( + ctx context.Context, + eg *errgroup.Group, + scheduler *regionRequestScheduler, + storeAddr string, +) *requestedStore { + pendingRegionRequestQueueSize := scheduler.config.PendingRegionRequestQueueSize + regionRequestWorkerPerStore := scheduler.config.RegionRequestWorkerPerStore + perWorkerQueueSize := pendingRegionRequestQueueSize / int(regionRequestWorkerPerStore) + if perWorkerQueueSize <= 0 { + log.Warn("pending region request queue size is smaller than the number of workers, adjust per worker queue size to 1", + zap.Int("pendingRegionRequestQueueSize", pendingRegionRequestQueueSize), + zap.Uint("regionRequestWorkerPerStore", regionRequestWorkerPerStore)) + perWorkerQueueSize = 1 + } + perStoreQuotaSize := perWorkerQueueSize * int(regionRequestWorkerPerStore) + + rs := &requestedStore{ + scheduler: scheduler, + storeAddr: storeAddr, + requestWorkers: make([]*regionRequestWorker, 0, regionRequestWorkerPerStore), + pendingTasks: priorityqueue.New[*regionPriorityTask](), + } + rs.quota = newStoreQuota(perStoreQuotaSize, rs.NotifyAvailable) + for range regionRequestWorkerPerStore { + requestWorker := newRegionRequestWorker( + ctx, + eg, + rs, + perWorkerQueueSize, + scheduler.upstream, + scheduler.eventSink, + scheduler.failureHandler, + ) + rs.requestWorkers = append(rs.requestWorkers, requestWorker) + } + return rs +} + +func (rs *requestedStore) Close() { + for _, worker := range rs.requestWorkers { + worker.ClearRegionRequests() + } +} + +func (rs *requestedStore) AddRegion( + ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, +) (bool, *regionRequestWorker, error) { + workers := rs.requestWorkers + + start := int(rs.nextWorker.Add(1)) % len(workers) + for i := range len(workers) { + worker := workers[(start+i)%len(workers)] + ok, err := worker.AddRegionRequest(ctx, region, force, quota) + if err != nil || ok { + return ok, worker, err + } + } + return false, nil, nil +} + +func (rs *requestedStore) NotifyAvailable() { + rs.notifyMu.Lock() + if rs.notifyEnqueued { + rs.notifyMu.Unlock() + return + } + rs.notifyEnqueued = true + rs.notifyMu.Unlock() + + rs.scheduler.storeAvailable.Push(rs) + rs.scheduler.notifyScheduler() +} + +func (rs *requestedStore) MarkAvailableDequeued() { + rs.notifyMu.Lock() + rs.notifyEnqueued = false + rs.notifyMu.Unlock() +} diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index b557b100f9..a42f586b1d 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -499,7 +499,7 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - ok, err := worker.Add(ctx, dummyRegion, true, testRegionRequestQuota()) + ok, err := worker.AddRegionRequest(ctx, dummyRegion, true, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) @@ -509,7 +509,7 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { require.True(t, ok) require.Equal(t, SubscriptionID(1), req.subID) require.True(t, req.filterLoop) - require.Equal(t, 1, worker.requestCache.PendingCount()) + require.Equal(t, 1, worker.PendingRequestCount()) } func TestRequestedStoreDeferredTasksPriority(t *testing.T) { From 945504c51be74e31665d56aa91d9bbd66b7052f2 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Tue, 30 Jun 2026 14:38:50 -0700 Subject: [PATCH 19/21] refactor --- logservice/logpuller/region_event_handler.go | 2 +- .../logpuller/region_event_handler_test.go | 18 +- logservice/logpuller/region_request_cache.go | 141 ++++----- .../logpuller/region_request_cache_test.go | 38 ++- .../logpuller/region_request_scheduler.go | 5 +- logservice/logpuller/region_request_worker.go | 278 ++++++------------ .../logpuller/region_request_worker_test.go | 126 ++++---- logservice/logpuller/region_state.go | 49 +-- logservice/logpuller/region_tracker.go | 134 +++++++++ logservice/logpuller/requested_store.go | 26 +- .../logpuller/subscription_client_test.go | 9 +- utils/notifyqueue/notify_queue.go | 77 +++++ utils/notifyqueue/notify_queue_test.go | 83 ++++++ 13 files changed, 616 insertions(+), 370 deletions(-) create mode 100644 logservice/logpuller/region_tracker.go create mode 100644 utils/notifyqueue/notify_queue.go create mode 100644 utils/notifyqueue/notify_queue_test.go diff --git a/logservice/logpuller/region_event_handler.go b/logservice/logpuller/region_event_handler.go index 777af088bd..91c4466bd0 100644 --- a/logservice/logpuller/region_event_handler.go +++ b/logservice/logpuller/region_event_handler.go @@ -255,7 +255,7 @@ func (h *regionEventHandler) handleRegionError(state *regionFeedState) { zap.Error(err)) } if stepsToRemoved { - worker.takeRegionState(SubscriptionID(state.requestID), state.getRegionID()) + worker.tracker.RemoveRegion(SubscriptionID(state.requestID), state.getRegionID()) h.failureHandler.Report(newRegionErrorInfo(state.getRegionInfo(), err)) } } diff --git a/logservice/logpuller/region_event_handler_test.go b/logservice/logpuller/region_event_handler_test.go index e94d176ed0..daca162f77 100644 --- a/logservice/logpuller/region_event_handler_test.go +++ b/logservice/logpuller/region_event_handler_test.go @@ -79,6 +79,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { worker := ®ionRequestWorker{ requestCache: &requestCache{}, + tracker: newRegionTracker(0), } region := newRegionInfo( tikv.RegionVerID{}, @@ -88,8 +89,7 @@ func TestHandleEventEntryEventOutOfOrder(t *testing.T) { false, ) region.lockedRangeState = ®ionlock.LockedRangeState{} - state := newRegionFeedState(region, 1, worker) - state.start() + state := newRegionFeedState(region, 1, worker, nil) // Receive prewrite2 with empty value. { @@ -218,9 +218,9 @@ func TestHandleResolvedTs(t *testing.T) { subID1 := SubscriptionID(1) worker := ®ionRequestWorker{ requestCache: &requestCache{}, + tracker: newRegionTracker(0), } - state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker) - state1.start() + state1 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(1, 1, 1)}, uint64(subID1), worker, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -243,8 +243,7 @@ func TestHandleResolvedTs(t *testing.T) { } subID2 := SubscriptionID(2) - state2 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(2, 2, 2)}, uint64(subID2), worker) - state2.start() + state2 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(2, 2, 2)}, uint64(subID2), worker, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -267,8 +266,7 @@ func TestHandleResolvedTs(t *testing.T) { } subID3 := SubscriptionID(3) - state3 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(3, 3, 3)}, uint64(subID3), worker) - state3.start() + state3 := newRegionFeedState(regionInfo{verID: tikv.NewRegionVerID(3, 3, 3)}, uint64(subID3), worker, nil) { span := heartbeatpb.TableSpan{ TableID: 100, @@ -361,6 +359,8 @@ func TestHandleResolvedTsThrottled(t *testing.T) { advanceInterval: 100, } span.lastAdvanceTime.Store(0) + worker := ®ionRequestWorker{} + worker.tracker = newRegionTracker(0) state := newRegionFeedState( regionInfo{ verID: tikv.NewRegionVerID(1, 1, 1), @@ -368,9 +368,9 @@ func TestHandleResolvedTsThrottled(t *testing.T) { lockedRangeState: res1.LockedRangeState, }, 1, + worker, nil, ) - state.start() require.Equal(t, uint64(200), handleResolvedTs(span, state, 300)) } diff --git a/logservice/logpuller/region_request_cache.go b/logservice/logpuller/region_request_cache.go index 6d77409fcc..535d01408d 100644 --- a/logservice/logpuller/region_request_cache.go +++ b/logservice/logpuller/region_request_cache.go @@ -14,13 +14,13 @@ package logpuller import ( - "container/list" "context" "sync" "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" + "github.com/pingcap/ticdc/utils/notifyqueue" "go.uber.org/zap" ) @@ -44,7 +44,10 @@ const ( regionReqStageSent ) -// regionReq tracks one data request from admission to completion. +// regionReq tracks one region request from admission to cleanup. +// +// The request also owns the region scan quota. The quota is released when the +// scan is initialized or the request is aborted before scan initialization. type regionReq struct { regionInfo regionInfo createTime time.Time @@ -53,65 +56,41 @@ type regionReq struct { // when the request leaves the worker window. quota *regionRequestQuota - // cache is set by requestCache.add. It lets regionFeedState finish exactly - // the request that created it instead of looking up by subID/regionID. - cache *requestCache - // readyElement is set while the request is waiting in requestCache.ready. - readyElement *list.Element // stage is guarded by requestCache.mu. stage regionReqStage } -func newRegionReq(cache *requestCache, region regionInfo, quota *regionRequestQuota) *regionReq { +func newRegionReq(region regionInfo, quota *regionRequestQuota) *regionReq { return ®ionReq{ regionInfo: region, createTime: time.Now(), quota: quota, - cache: cache, stage: regionReqStageQueued, } } -func (r *regionReq) MarkSent() { - r.cache.markSent(r) -} - -func (r *regionReq) Resolve() bool { - return r.cache.resolve(r) -} - -func (r *regionReq) Finish() bool { - return r.cache.finish(r) -} - -func (r *regionReq) ReleaseQuota() { - r.quota.Release() -} - -// requestCache manages worker-local data requests with flow control. +// requestCache manages worker-local region requests with flow control. // // requests is the source of truth for live requests. A request is inserted by -// add(), moves through queued/processing/sent, and is removed by resolve(), -// finish(), takeUnsentRegions(), or clear(). +// add(), moves through queued/processing/sent, and is removed by finishScan(), +// abortScan(), drainUnsentRegions(), or close(). type requestCache struct { mu sync.Mutex - // requests owns every live data request in this worker. Its length is the - // flow-control count used by add(), pendingCount(), and metrics. - requests map[*regionReq]struct{} - - // ready is the FIFO list of queued requests. - ready *list.List - // maxPendingCount limits len(requests) for non-force adds. force adds bypass // this limit, matching the old force behavior for high-priority data requests. maxPendingCount int - // readyAvailable wakes a worker blocked in pop() when a queued request is - // appended. It is a level-trigger hint; callers must re-check ready under mu. - readyAvailable chan struct{} + // requests owns every live region request in this worker. Its length is the + // flow-control count used by add(), pendingCount(), and metrics. + requests map[*regionReq]struct{} + + // queue stores queued requests in FIFO order. + queue *notifyqueue.Queue[*regionReq] + // spaceAvailable wakes add() when a live request leaves requests. spaceAvailable chan struct{} + // onSpaceAvailable lets the store-level deferred scheduler retry a task when // this worker frees request capacity. onSpaceAvailable func() @@ -119,16 +98,15 @@ type requestCache struct { func newRequestCache(maxPendingCount int, onSpaceAvailable func()) *requestCache { return &requestCache{ - requests: make(map[*regionReq]struct{}), - ready: list.New(), maxPendingCount: maxPendingCount, - readyAvailable: make(chan struct{}, 1), + requests: make(map[*regionReq]struct{}), + queue: notifyqueue.New[*regionReq](), spaceAvailable: make(chan struct{}, 1), onSpaceAvailable: onSpaceAvailable, } } -// add admits a data request into the worker window. +// add admits a region request into the worker window. func (c *requestCache) add( ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, ) (bool, error) { @@ -157,24 +135,16 @@ func (c *requestCache) add( } func (c *requestCache) tryAdd(region regionInfo, force bool, quota *regionRequestQuota) bool { - notifyReady := false - c.mu.Lock() - defer func() { - c.mu.Unlock() - if notifyReady { - c.notifyReady() - } - }() + defer c.mu.Unlock() if len(c.requests) >= c.maxPendingCount && !force { return false } - req := newRegionReq(c, region, quota) + req := newRegionReq(region, quota) c.requests[req] = struct{}{} - req.readyElement = c.ready.PushBack(req) - notifyReady = true + c.queue.Push(req) return true } @@ -186,7 +156,7 @@ func (c *requestCache) pop(ctx context.Context) (*regionReq, error) { } select { - case <-c.readyAvailable: + case <-c.queue.Ready(): case <-ctx.Done(): return nil, ctx.Err() } @@ -197,26 +167,39 @@ func (c *requestCache) tryPop() *regionReq { c.mu.Lock() defer c.mu.Unlock() - elem := c.ready.Front() - if elem == nil { - return nil + for { + req, ok := c.queue.TryPop() + if !ok { + return nil + } + if _, ok := c.requests[req]; !ok { + log.Warn("request cache pops a removed request", + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID()), + zap.Uint8("stage", uint8(req.stage))) + continue + } + if req.stage != regionReqStageQueued { + log.Warn("request cache pops a non-queued request", + zap.Uint64("subID", uint64(req.regionInfo.subscribedSpan.subID)), + zap.Uint64("regionID", req.regionInfo.verID.GetID()), + zap.Uint8("stage", uint8(req.stage))) + continue + } + req.stage = regionReqStageProcessing + return req } - req := elem.Value.(*regionReq) - c.ready.Remove(elem) - req.readyElement = nil - req.stage = regionReqStageProcessing - return req } func (c *requestCache) markSent(req *regionReq) { c.mu.Lock() + defer c.mu.Unlock() if _, ok := c.requests[req]; ok && req.stage == regionReqStageProcessing { req.stage = regionReqStageSent } - c.mu.Unlock() } -func (c *requestCache) resolve(req *regionReq) bool { +func (c *requestCache) finishScan(req *regionReq) bool { if !c.remove(req) { return false } @@ -237,7 +220,7 @@ func (c *requestCache) resolve(req *regionReq) bool { return true } -func (c *requestCache) finish(req *regionReq) bool { +func (c *requestCache) abortScan(req *regionReq) bool { return c.remove(req) } @@ -251,13 +234,15 @@ func (c *requestCache) remove(req *regionReq) bool { c.mu.Unlock() if removed { - req.ReleaseQuota() + req.quota.Release() c.notifySpace() } return removed } -func (c *requestCache) takeUnsentRegions() []regionInfo { +// drainUnsentRegions removes queued and processing requests and returns their +// regions. Sent requests are owned by regionTracker and are not drained here. +func (c *requestCache) drainUnsentRegions() []regionInfo { c.mu.Lock() removedReqs := make([]*regionReq, 0, len(c.requests)) regions := make([]regionInfo, 0, len(c.requests)) @@ -270,33 +255,32 @@ func (c *requestCache) takeUnsentRegions() []regionInfo { regions = append(regions, req.regionInfo) } } + c.queue.Drain() c.mu.Unlock() c.releaseRemovedReqs(removedReqs) return regions } -// clear removes all live requests and returns their regions. -func (c *requestCache) clear() []regionInfo { +// close removes all live requests when the worker is closed. +func (c *requestCache) close() { c.mu.Lock() removedReqs := make([]*regionReq, 0, len(c.requests)) - regions := make([]regionInfo, 0, len(c.requests)) for req := range c.requests { if c.removeLocked(req) { removedReqs = append(removedReqs, req) - regions = append(regions, req.regionInfo) } } + c.queue.Drain() c.mu.Unlock() c.releaseRemovedReqs(removedReqs) - return regions } func (c *requestCache) releaseRemovedReqs(removedReqs []*regionReq) { if len(removedReqs) > 0 { for _, req := range removedReqs { - req.ReleaseQuota() + req.quota.Release() } c.notifySpace() } @@ -318,18 +302,11 @@ func (c *requestCache) removeLocked(req *regionReq) bool { } delete(c.requests, req) - if req.readyElement != nil { - c.ready.Remove(req.readyElement) - req.readyElement = nil - } return true } -func (c *requestCache) notifyReady() { - select { - case c.readyAvailable <- struct{}{}: - default: - } +func (c *requestCache) ready() <-chan struct{} { + return c.queue.Ready() } func (c *requestCache) notifySpace() { diff --git a/logservice/logpuller/region_request_cache_test.go b/logservice/logpuller/region_request_cache_test.go index fbe8614a17..c600fe8566 100644 --- a/logservice/logpuller/region_request_cache_test.go +++ b/logservice/logpuller/region_request_cache_test.go @@ -95,10 +95,10 @@ func TestRequestCacheAdd_ForceFlag(t *testing.T) { require.Equal(t, region1.verID.GetID(), req.regionInfo.verID.GetID()) require.Equal(t, region1.subscribedSpan.subID, req.regionInfo.subscribedSpan.subID) require.Equal(t, 2, cache.pendingCount()) - req.MarkSent() + cache.markSent(req) // resolve region1 - req.Resolve() + cache.finishScan(req) require.Equal(t, 1, cache.pendingCount()) } @@ -162,11 +162,11 @@ func TestRequestCacheAdd_SpaceAvailableNotification(t *testing.T) { require.NoError(t, err) require.NotNil(t, req) require.Equal(t, 2, cache.pendingCount()) // pop doesn't change pendingCount - req.MarkSent() + cache.markSent(req) require.Equal(t, 2, cache.pendingCount()) // Resolve the request to free up space - success := req.Resolve() + success := cache.finishScan(req) require.True(t, success) require.Equal(t, 1, cache.pendingCount()) @@ -231,7 +231,7 @@ func TestRequestCacheAdd_DuplicateQueuedRequestsAreTrackedIndependently(t *testi require.NotSame(t, req1, req2) } -func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { +func TestRequestCacheAbortReleasesSlot(t *testing.T) { cache := newRequestCache(10, nil) ctx := context.Background() @@ -245,14 +245,14 @@ func TestRequestCacheFinish_ReleasesSlot(t *testing.T) { req, err := cache.pop(ctx) require.NoError(t, err) - req.MarkSent() + cache.markSent(req) require.Equal(t, 1, cache.pendingCount()) - req.Finish() + cache.abortScan(req) require.Equal(t, 0, cache.pendingCount()) } -func TestRequestCacheFinishReleasesQuotaOnce(t *testing.T) { +func TestRequestCacheAbortReleasesQuotaOnce(t *testing.T) { cache := newRequestCache(10, nil) ctx := context.Background() region := createTestRegionInfo(1, 1) @@ -269,7 +269,25 @@ func TestRequestCacheFinishReleasesQuotaOnce(t *testing.T) { req, err := cache.pop(ctx) require.NoError(t, err) - require.True(t, req.Finish()) - require.False(t, req.Finish()) + require.True(t, cache.abortScan(req)) + require.False(t, cache.abortScan(req)) require.Equal(t, int32(1), releaseCount.Load()) } + +func TestRequestCacheCloseDrainsQueuedRequests(t *testing.T) { + cache := newRequestCache(10, nil) + ctx := context.Background() + + ok, err := cache.add(ctx, createTestRegionInfo(1, 1), false, testRegionRequestQuota()) + require.NoError(t, err) + require.True(t, ok) + ok, err = cache.add(ctx, createTestRegionInfo(1, 2), false, testRegionRequestQuota()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 2, cache.queue.Len()) + + cache.close() + require.Equal(t, 0, cache.pendingCount()) + require.Equal(t, 0, cache.queue.Len()) + require.Nil(t, cache.tryPop()) +} diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index 66212ccf2d..b52065f92b 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -77,8 +77,9 @@ func (s *regionRequestScheduler) Run(ctx context.Context, eg *errgroup.Group) er return rs } - rs = newRequestedStore(ctx, eg, s, storeAddr) + rs = newRequestedStore(s, storeAddr) s.stores.Store(storeAddr, rs) + rs.Run(ctx, eg) return rs } @@ -144,7 +145,7 @@ func (s *regionRequestScheduler) UpdateMetrics() { s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) for _, worker := range store.requestWorkers { - count += worker.PendingRequestCount() + count += worker.requestCache.pendingCount() } return true }) diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 272d63cc7d..4720ed2d44 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -26,61 +26,61 @@ import ( "github.com/pingcap/log" cerror "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/metrics" - "github.com/pingcap/ticdc/pkg/security" "github.com/pingcap/ticdc/pkg/util" "github.com/pingcap/ticdc/pkg/version" - "github.com/pingcap/ticdc/utils/chann" + "github.com/pingcap/ticdc/utils/notifyqueue" "go.uber.org/zap" "golang.org/x/sync/errgroup" grpcstatus "google.golang.org/grpc/status" ) +const storeReconnectBackoff = time.Second + // To generate a workerID in `newRegionRequestWorker`. var workerIDGen atomic.Uint64 -type regionFeedStates map[uint64]*regionFeedState - type deregisterRequest struct { subID SubscriptionID filterLoop bool } type controlQueue struct { - ch *chann.UnlimitedChannel[deregisterRequest, any] - notify chan struct{} + mu sync.Mutex + queue *notifyqueue.Queue[deregisterRequest] } func newControlQueue() *controlQueue { return &controlQueue{ - ch: chann.NewUnlimitedChannelDefault[deregisterRequest](), - notify: make(chan struct{}, 1), + queue: notifyqueue.New[deregisterRequest](), } } func (q *controlQueue) push(req deregisterRequest) { - q.ch.Push(req) - select { - case q.notify <- struct{}{}: - default: - } + q.mu.Lock() + defer q.mu.Unlock() + q.queue.Push(req) } func (q *controlQueue) tryPop() (deregisterRequest, bool) { - if q.ch.Len() == 0 { - return deregisterRequest{}, false - } - req, ok, err := q.ch.GetWithContext(context.Background()) - if err != nil { - return deregisterRequest{}, false - } - return req, ok + q.mu.Lock() + defer q.mu.Unlock() + return q.queue.TryPop() +} + +func (q *controlQueue) len() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.queue.Len() +} + +func (q *controlQueue) ready() <-chan struct{} { + return q.queue.Ready() } // regionRequestWorker is responsible for sending region requests to a specific TiKV store. type regionRequestWorker struct { - workerID uint64 - - store *requestedStore + workerID uint64 + storeAddr string upstream *upstreamHandle eventSink *regionEventSink @@ -90,60 +90,73 @@ type regionRequestWorker struct { requestCache *requestCache controlQueue *controlQueue - // all regions maintained by this worker. - requestedRegions struct { - sync.RWMutex - - subscriptions map[SubscriptionID]regionFeedStates - } + tracker *regionTracker } func newRegionRequestWorker( - ctx context.Context, - g *errgroup.Group, - store *requestedStore, - requestCacheSize int, + storeAddr string, + requestCache *requestCache, upstream *upstreamHandle, eventSink *regionEventSink, failureHandler *regionFailureHandler, ) *regionRequestWorker { + workerID := workerIDGen.Add(1) worker := ®ionRequestWorker{ - workerID: workerIDGen.Add(1), - store: store, + workerID: workerID, + storeAddr: storeAddr, upstream: upstream, eventSink: eventSink, failureHandler: failureHandler, - requestCache: newRequestCache(requestCacheSize, func() { - store.NotifyAvailable() - }), - controlQueue: newControlQueue(), + requestCache: requestCache, + controlQueue: newControlQueue(), + tracker: newRegionTracker(workerID), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) + return worker +} - g.Go(func() error { - for { - firstReq, err := worker.waitForRegion(ctx) - if err != nil { - return err - } - err = worker.checkStoreVersion(ctx) - if err == nil { - err = worker.Run(ctx, worker.upstream.credential, firstReq) - } - if ctx.Err() != nil { - return ctx.Err() - } - if err == nil { - err = &storeStreamErr{} - } - worker.handleStoreFailure(err) - if err := util.Hang(ctx, time.Second); err != nil { - return err +func (s *regionRequestWorker) Run(ctx context.Context) error { + for { + // Wait for one data request before connecting, so an idle worker does + // not keep reconnecting to an unavailable store. + firstReq, err := s.waitForRegion(ctx) + if err != nil { + return err + } + + // Run one stream session. A clean stream exit is still treated as a + // store failure so tracked regions can be retried. + var regionErr error = &storeStreamErr{} + if err := s.checkStoreVersion(ctx); err != nil { + regionErr = err + } else if err := s.runStream(ctx, firstReq); err != nil { + regionErr = err + } + if ctx.Err() != nil { + return ctx.Err() + } + + // Stop all tracked regions for this stream and send events so the event + // handler can reschedule them through the normal failure path. + for subID, states := range s.tracker.Drain() { + for _, state := range states { + state.markStopped(regionErr) + regionEvent := regionEvent{ + states: []*regionFeedState{state}, + } + s.eventSink.Push(subID, regionEvent) } } - }) + // Unsent regions are not tracked yet, so report them through the + // failure path explicitly. + for _, region := range s.requestCache.drainUnsentRegions() { + s.failureHandler.Report(newRegionErrorInfo(region, regionErr)) + } - return worker + // Avoid a tight reconnect loop when the store keeps failing. + if err := util.Hang(ctx, storeReconnectBackoff); err != nil { + return err + } + } } func (s *regionRequestWorker) waitForRegion(ctx context.Context) (*regionReq, error) { @@ -164,7 +177,7 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { } log.Error("event feed check store version fails", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) if cerror.Is(err, cerror.ErrGetAllStoresFailed) { return &getStoreErr{} @@ -172,42 +185,24 @@ func (s *regionRequestWorker) checkStoreVersion(ctx context.Context) error { return &storeStreamErr{} } -func (s *regionRequestWorker) handleStoreFailure(regionErr error) { - for subID, m := range s.clearRegionStates() { - for _, state := range m { - state.markStopped(regionErr) - regionEvent := regionEvent{ - states: []*regionFeedState{state}, - } - s.eventSink.Push(subID, regionEvent) - } - } - // The store may fail forever, so we need try to re-schedule all pending regions. - for _, region := range s.clearPendingRegions() { - s.failureHandler.Report(newRegionErrorInfo(region, regionErr)) - } -} - -func (s *regionRequestWorker) Run( - ctx context.Context, credential *security.Credential, firstReq *regionReq, -) (err error) { +func (s *regionRequestWorker) runStream(ctx context.Context, firstReq *regionReq) (err error) { log.Info("region request worker going to create grpc stream", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr)) + zap.String("addr", s.storeAddr)) defer func() { log.Info("region request worker exits", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) }() g, gctx := errgroup.WithContext(ctx) - conn, err := Connect(gctx, credential, s.store.storeAddr) + conn, err := Connect(gctx, s.upstream.credential, s.storeAddr) if err != nil { log.Warn("region request worker create grpc stream failed", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) // Close the connection if it was partially created to prevent goroutine leaks if conn != nil && conn.Conn != nil { @@ -261,7 +256,7 @@ func (s *regionRequestWorker) receiveAndDispatchChangeEvents(conn *ConnAndClient if err != nil { log.Info("region request worker receive from grpc stream failed", zap.Uint64("workerID", s.workerID), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.String("code", grpcstatus.Code(err).String()), zap.Error(err)) return normalizeStreamError(err) @@ -279,7 +274,7 @@ func (s *regionRequestWorker) dispatchRegionChangeEvents(events []*cdcpb.Event) for _, event := range events { regionID := event.RegionId subscriptionID := SubscriptionID(event.RequestId) - state := s.getRegionState(subscriptionID, regionID) + state := s.tracker.Get(subscriptionID, regionID) if state != nil { regionEvent := regionEvent{ states: []*regionFeedState{state}, @@ -362,7 +357,7 @@ func (s *regionRequestWorker) dispatchResolvedTsEvent(resolvedTsEvent *cdcpb.Res resolvedStates = nil } for i, regionID := range resolvedTsEvent.Regions { - if state := s.getRegionState(subscriptionID, regionID); state != nil { + if state := s.tracker.Get(subscriptionID, regionID); state != nil { resolvedStates = append(resolvedStates, state) if len(resolvedStates) >= resolvedTsStateBatchSize { flush() @@ -397,7 +392,7 @@ func (s *regionRequestWorker) processRegionSendTask( zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", req.RequestId), zap.Uint64("regionID", req.RegionId), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Error(err)) return normalizeStreamError(err) } @@ -416,7 +411,7 @@ func (s *regionRequestWorker) processRegionSendTask( if err := doSend(changeDataReq); err != nil { return err } - for _, state := range s.takeRegionStates(req.subID) { + for _, state := range s.tracker.RemoveSubscription(req.subID) { state.markStopped(&requestCancelledErr{}) regionEvent := regionEvent{ states: []*regionFeedState{state}, @@ -446,20 +441,19 @@ func (s *regionRequestWorker) processRegionSendTask( zap.Uint64("workerID", s.workerID), zap.Uint64("subscriptionID", uint64(subID)), zap.Uint64("regionID", region.verID.GetID()), - zap.String("addr", s.store.storeAddr), + zap.String("addr", s.storeAddr), zap.Bool("bdrMode", region.filterLoop)) if region.subscribedSpan.stopped.Load() { // The subscription has been stopped before this queued request is sent. s.failureHandler.Report(newRegionErrorInfo(region, &storeStreamErr{})) - regionReq.Finish() + s.requestCache.abortScan(regionReq) } else { state := newRegionFeedState(region, uint64(subID), s, regionReq) - state.start() - s.addRegionState(subID, region.verID.GetID(), state) + s.tracker.Track(subID, region.verID.GetID(), state) // Mark the request as sent before sending it. // Otherwise there is a race with the receiver goroutine: - // 1. addRegionState makes the region visible to error handling. + // 1. tracker.Track makes the region visible to error handling. // 2. doSend sends the request. // 3. the receiver goroutine may receive a region error immediately. // 4. markStopped runs before markSent, so the request may be finished @@ -467,11 +461,11 @@ func (s *regionRequestWorker) processRegionSendTask( // 5. the sender goroutine then calls markSent and must not make the // finished request live again. // - // Tracking the request before Send keeps requestedRegions and + // Tracking the request before Send keeps regionTracker and // request lifecycle visible in the same order and avoids leaving stale // requests behind. - regionReq.MarkSent() - if err := doSend(s.createRegionRequest(region)); err != nil { + s.requestCache.markSent(regionReq) + if err := doSend(createRegionRequest(s.upstream.clusterID, region)); err != nil { state.markStopped(err) return err } @@ -487,17 +481,17 @@ func (s *regionRequestWorker) processRegionSendTask( continue } select { - case <-s.controlQueue.notify: - case <-s.requestCache.readyAvailable: + case <-s.controlQueue.ready(): + case <-s.requestCache.ready(): case <-ctx.Done(): return ctx.Err() } } } -func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.ChangeDataRequest { +func createRegionRequest(clusterID uint64, region regionInfo) *cdcpb.ChangeDataRequest { return &cdcpb.ChangeDataRequest{ - Header: &cdcpb.Header{ClusterId: s.upstream.clusterID, TicdcVersion: version.ReleaseSemver()}, + Header: &cdcpb.Header{ClusterId: clusterID, TicdcVersion: version.ReleaseSemver()}, RegionId: region.verID.GetID(), RequestId: uint64(region.subscribedSpan.subID), RegionEpoch: region.rpcCtx.Meta.RegionEpoch, @@ -508,81 +502,3 @@ func (s *regionRequestWorker) createRegionRequest(region regionInfo) *cdcpb.Chan FilterLoop: region.filterLoop, } } - -func (s *regionRequestWorker) addRegionState(subscriptionID SubscriptionID, regionID uint64, state *regionFeedState) { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - states := s.requestedRegions.subscriptions[subscriptionID] - if states == nil { - states = make(regionFeedStates) - s.requestedRegions.subscriptions[subscriptionID] = states - } - - if oldState := states[regionID]; oldState != nil && oldState.request != state.request { - log.Warn("region request state overwritten", - zap.Uint64("workerID", s.workerID), - zap.Uint64("subscriptionID", uint64(subscriptionID)), - zap.Uint64("regionID", regionID)) - if oldState.request != nil { - oldState.request.Finish() - } - } - states[regionID] = state -} - -func (s *regionRequestWorker) getRegionState(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { - s.requestedRegions.RLock() - defer s.requestedRegions.RUnlock() - if states, ok := s.requestedRegions.subscriptions[subscriptionID]; ok { - return states[regionID] - } - return nil -} - -func (s *regionRequestWorker) takeRegionState(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - if statesMap, ok := s.requestedRegions.subscriptions[subscriptionID]; ok { - state := statesMap[regionID] - delete(statesMap, regionID) - if len(statesMap) == 0 { - delete(s.requestedRegions.subscriptions, subscriptionID) - } - return state - } - return nil -} - -func (s *regionRequestWorker) takeRegionStates(subscriptionID SubscriptionID) regionFeedStates { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - states := s.requestedRegions.subscriptions[subscriptionID] - delete(s.requestedRegions.subscriptions, subscriptionID) - return states -} - -func (s *regionRequestWorker) clearRegionStates() map[SubscriptionID]regionFeedStates { - s.requestedRegions.Lock() - defer s.requestedRegions.Unlock() - subscriptions := s.requestedRegions.subscriptions - s.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - return subscriptions -} - -func (s *regionRequestWorker) AddRegionRequest( - ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, -) (bool, error) { - return s.requestCache.add(ctx, region, force, quota) -} - -func (s *regionRequestWorker) ClearRegionRequests() { - s.requestCache.clear() -} - -func (s *regionRequestWorker) PendingRequestCount() int { - return s.requestCache.pendingCount() -} - -func (s *regionRequestWorker) clearPendingRegions() []regionInfo { - return s.requestCache.takeUnsentRegions() -} diff --git a/logservice/logpuller/region_request_worker_test.go b/logservice/logpuller/region_request_worker_test.go index b776c559fe..7164a372c6 100644 --- a/logservice/logpuller/region_request_worker_test.go +++ b/logservice/logpuller/region_request_worker_test.go @@ -57,30 +57,28 @@ func prepareRegionForSendTest(region regionInfo) regionInfo { } func TestRegionStatesOperation(t *testing.T) { - worker := ®ionRequestWorker{} - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) - - require.Nil(t, worker.getRegionState(1, 2)) - require.Nil(t, worker.takeRegionState(1, 2)) - - worker.addRegionState(1, 2, ®ionFeedState{}) - require.NotNil(t, worker.getRegionState(1, 2)) - require.NotNil(t, worker.takeRegionState(1, 2)) - require.Nil(t, worker.getRegionState(1, 2)) - require.Equal(t, 0, len(worker.requestedRegions.subscriptions)) - - worker.addRegionState(1, 2, ®ionFeedState{}) - require.NotNil(t, worker.getRegionState(1, 2)) - require.NotNil(t, worker.takeRegionState(1, 2)) - require.Nil(t, worker.getRegionState(1, 2)) - require.Equal(t, 0, len(worker.requestedRegions.subscriptions)) + tracker := newRegionTracker(0) + + require.Nil(t, tracker.Get(1, 2)) + require.Nil(t, tracker.RemoveRegion(1, 2)) + + tracker.Track(1, 2, ®ionFeedState{}) + require.NotNil(t, tracker.Get(1, 2)) + require.NotNil(t, tracker.RemoveRegion(1, 2)) + require.Nil(t, tracker.Get(1, 2)) + + tracker.Track(1, 2, ®ionFeedState{}) + require.NotNil(t, tracker.Get(1, 2)) + require.NotNil(t, tracker.RemoveRegion(1, 2)) + require.Nil(t, tracker.Get(1, 2)) } func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { + requestCache := newRequestCache(10, nil) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10, nil), + requestCache: requestCache, + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) ctx := context.Background() region := createTestRegionInfo(1, 1) @@ -90,9 +88,9 @@ func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { require.True(t, ok) req1, err := worker.requestCache.pop(ctx) require.NoError(t, err) - req1.MarkSent() + worker.requestCache.markSent(req1) state1 := newRegionFeedState(req1.regionInfo, uint64(req1.regionInfo.subscribedSpan.subID), worker, req1) - worker.addRegionState(req1.regionInfo.subscribedSpan.subID, req1.regionInfo.verID.GetID(), state1) + worker.tracker.Track(req1.regionInfo.subscribedSpan.subID, req1.regionInfo.verID.GetID(), state1) ok, err = worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) require.NoError(t, err) @@ -101,13 +99,41 @@ func TestAddRegionStateReleasesOverwrittenRequest(t *testing.T) { req2, err := worker.requestCache.pop(ctx) require.NoError(t, err) state2 := newRegionFeedState(req2.regionInfo, uint64(req2.regionInfo.subscribedSpan.subID), worker, req2) - worker.addRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID(), state2) + worker.tracker.Track(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID(), state2) require.Equal(t, 1, worker.requestCache.pendingCount()) - require.Same(t, state2, worker.getRegionState(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID())) + require.Same(t, state2, worker.tracker.Get(req2.regionInfo.subscribedSpan.subID, req2.regionInfo.verID.GetID())) } -func TestClearPendingRegionsReleaseSlotForProcessingRegion(t *testing.T) { +func TestInitializedRegionStateDetachesRequest(t *testing.T) { + requestCache := newRequestCache(10, nil) + worker := ®ionRequestWorker{ + requestCache: requestCache, + tracker: newRegionTracker(0), + } + + ctx := context.Background() + region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) + + ok, err := worker.requestCache.add(ctx, region, false, testRegionRequestQuota()) + require.NoError(t, err) + require.True(t, ok) + req, err := worker.requestCache.pop(ctx) + require.NoError(t, err) + worker.requestCache.markSent(req) + + state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) + worker.tracker.Track(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) + require.Equal(t, 1, worker.requestCache.pendingCount()) + + state.setInitialized() + require.Equal(t, 0, worker.requestCache.pendingCount()) + + state.markStopped(errors.New("region stopped after initialized")) + require.Equal(t, 0, worker.requestCache.pendingCount()) +} + +func TestDrainUnsentRegionsReleaseSlotForProcessingRegion(t *testing.T) { worker := ®ionRequestWorker{ requestCache: newRequestCache(10, nil), } @@ -123,7 +149,7 @@ func TestClearPendingRegionsReleaseSlotForProcessingRegion(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, worker.requestCache.pendingCount()) - regions := worker.clearPendingRegions() + regions := worker.requestCache.drainUnsentRegions() require.Len(t, regions, 1) require.Equal(t, 0, worker.requestCache.pendingCount()) } @@ -180,17 +206,13 @@ func newDispatchResolvedTsTestWorker(regionCount int) (*regionRequestWorker, *mo ds := &mockRegionEventDynamicStream{} worker := ®ionRequestWorker{ eventSink: ®ionEventSink{ctx: context.Background(), ds: ds}, - } - worker.requestedRegions.subscriptions = map[SubscriptionID]regionFeedStates{ - 1: make(regionFeedStates, regionCount), + tracker: newRegionTracker(0), } regions := make([]uint64, regionCount) for i := 0; i < regionCount; i++ { regionID := uint64(i + 1) regions[i] = regionID - worker.requestedRegions.subscriptions[1][regionID] = ®ionFeedState{ - requestID: 1, - } + worker.tracker.Track(1, regionID, ®ionFeedState{requestID: 1}) } return worker, ds, &cdcpb.ResolvedTs{ @@ -216,7 +238,7 @@ func dispatchResolvedTsEventLegacyForBenchmark(s *regionRequestWorker, resolvedT resolvedStates = make([]*regionFeedState, 0, resolvedTsStateBatchSize) } for _, regionID := range resolvedTsEvent.Regions { - if state := s.getRegionState(subscriptionID, regionID); state != nil { + if state := s.tracker.Get(subscriptionID, regionID); state != nil { resolvedStates = append(resolvedStates, state) if len(resolvedStates) >= resolvedTsStateBatchSize { flush() @@ -290,11 +312,12 @@ func BenchmarkDispatchResolvedTsEventSmallBatchCurrent(b *testing.B) { benchmarkDispatchResolvedTsEvent(b, 16, false) } -func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { +func TestDrainUnsentRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { + requestCache := newRequestCache(10, nil) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10, nil), + requestCache: requestCache, + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) ctx := context.Background() region := createTestRegionInfo(1, 1) @@ -307,31 +330,31 @@ func TestClearPendingRegionsDoesNotReturnStoppedSentRegion(t *testing.T) { require.NoError(t, err) state := newRegionFeedState(req.regionInfo, uint64(req.regionInfo.subscribedSpan.subID), worker, req) - state.start() - worker.addRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) + worker.tracker.Track(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID(), state) // Simulate the race we are fixing in processRegionSendTask: once a request is // visible as sent, a fast region error may mark the region stopped before - // worker cleanup runs. In that case, markStopped should finish the request - // immediately, so clearPendingRegions must not return the stale region again. - req.MarkSent() + // worker cleanup runs. In that case, markStopped should abort the request + // immediately, so drainUnsentRegions must not return the stale region again. + worker.requestCache.markSent(req) state.markStopped(errors.New("send request to store error")) - worker.takeRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + worker.tracker.RemoveRegion(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.Equal(t, 0, worker.requestCache.pendingCount()) - require.Empty(t, worker.clearPendingRegions()) + require.Empty(t, worker.requestCache.drainUnsentRegions()) } func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { + requestCache := newRequestCache(10, nil) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10, nil), + requestCache: requestCache, controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{}, eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, failureHandler: newRegionFailureHandler(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) @@ -353,7 +376,7 @@ func TestProcessRegionSendTaskSendFailureCleansSentRequest(t *testing.T) { err = worker.processRegionSendTask(ctx, conn, req) require.ErrorIs(t, err, sendErr) require.Equal(t, 0, worker.requestCache.pendingCount()) - state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.True(t, state == nil || state.isStale(), "region state should be removed or marked stale after send failure") } @@ -374,15 +397,16 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + requestCache := newRequestCache(10, nil) worker := ®ionRequestWorker{ - requestCache: newRequestCache(10, nil), + requestCache: requestCache, controlQueue: newControlQueue(), - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", upstream: &upstreamHandle{}, eventSink: ®ionEventSink{ctx: context.Background(), ds: &mockRegionEventDynamicStream{}}, failureHandler: newRegionFailureHandler(&upstreamHandle{}, func(*subscribedSpan) {}, nil, nil), + tracker: newRegionTracker(0), } - worker.requestedRegions.subscriptions = make(map[SubscriptionID]regionFeedStates) ctx := context.Background() region := prepareRegionForSendTest(createTestRegionInfo(1, 1)) @@ -404,7 +428,7 @@ func TestProcessRegionSendTaskSendEOFIsRetriable(t *testing.T) { require.ErrorAs(t, err, &streamErr) require.Equal(t, 0, worker.requestCache.pendingCount()) - state := worker.getRegionState(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) + state := worker.tracker.Get(req.regionInfo.subscribedSpan.subID, req.regionInfo.verID.GetID()) require.NotNil(t, state) require.True(t, state.isStale()) @@ -432,7 +456,7 @@ func TestReceiveAndDispatchChangeEventsEOFIsRetriable(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { worker := ®ionRequestWorker{ - store: &requestedStore{storeAddr: "store-1"}, + storeAddr: "store-1", } conn := &ConnAndClient{ Client: &mockEventFeedV2Client{recvErr: tc.recvErr}, diff --git a/logservice/logpuller/region_state.go b/logservice/logpuller/region_state.go index 909b48e76e..9043748bdb 100644 --- a/logservice/logpuller/region_state.go +++ b/logservice/logpuller/region_state.go @@ -98,31 +98,30 @@ type regionFeedState struct { err error } - worker *regionRequestWorker - request *regionReq + regionReq struct { + sync.Mutex + request *regionReq + } + + worker *regionRequestWorker } func newRegionFeedState( region regionInfo, requestID uint64, worker *regionRequestWorker, - request ...*regionReq, + request *regionReq, ) *regionFeedState { state := ®ionFeedState{ region: region, requestID: requestID, + matcher: newMatcher(), worker: worker, } - if len(request) > 0 { - state.request = request[0] - } + state.regionReq.request = request return state } -func (s *regionFeedState) start() { - s.matcher = newMatcher() -} - // mark regionFeedState as stopped with the given error if possible. func (s *regionFeedState) markStopped(err error) { s.state.Lock() @@ -131,9 +130,7 @@ func (s *regionFeedState) markStopped(err error) { s.state.v = stateStopped s.state.err = err } - if s.request != nil { - s.request.Finish() - } + s.abortScanIfNeeded() } // mark regionFeedState as removed if possible. @@ -145,9 +142,7 @@ func (s *regionFeedState) markRemoved() (changed bool) { changed = true s.matcher.clear() } - if s.request != nil { - s.request.Finish() - } + s.abortScanIfNeeded() return } @@ -171,8 +166,26 @@ func (s *regionFeedState) isInitialized() bool { func (s *regionFeedState) setInitialized() { s.region.lockedRangeState.Initialized.Store(true) - if s.request != nil { - s.request.Resolve() + s.finishScan() +} + +func (s *regionFeedState) finishScan() { + s.regionReq.Lock() + request := s.regionReq.request + s.regionReq.request = nil + s.regionReq.Unlock() + + s.worker.requestCache.finishScan(request) +} + +func (s *regionFeedState) abortScanIfNeeded() { + s.regionReq.Lock() + request := s.regionReq.request + s.regionReq.request = nil + s.regionReq.Unlock() + + if request != nil { + s.worker.requestCache.abortScan(request) } } diff --git a/logservice/logpuller/region_tracker.go b/logservice/logpuller/region_tracker.go new file mode 100644 index 0000000000..07946cbc4d --- /dev/null +++ b/logservice/logpuller/region_tracker.go @@ -0,0 +1,134 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package logpuller + +import ( + "sync" + + "github.com/pingcap/log" + "go.uber.org/zap" +) + +type trackedRegionStates map[uint64]*regionFeedState + +// regionTracker is the regionRequestWorker-local state registry. +// +// It tracks only the regions sent by one worker. Normal scan request cleanup is +// owned by regionFeedState state transitions, so the tracker only indexes and +// removes states. The overwrite path aborts the old state's scan explicitly +// because no caller will transition that old state afterwards. +type regionTracker struct { + sync.RWMutex + workerID uint64 + + regionsBySubscription map[SubscriptionID]trackedRegionStates +} + +func newRegionTracker(workerID uint64) *regionTracker { + return ®ionTracker{ + workerID: workerID, + regionsBySubscription: make(map[SubscriptionID]trackedRegionStates), + } +} + +// Track records a region after the worker has picked a scan request from +// requestCache and before the request is sent to TiKV. +// +// If the same region is tracked again, the previous state's unfinished scan +// request is aborted before it can leak quota. +func (t *regionTracker) Track(subscriptionID SubscriptionID, regionID uint64, state *regionFeedState) { + t.Lock() + regions := t.regionsBySubscription[subscriptionID] + if regions == nil { + regions = make(trackedRegionStates) + t.regionsBySubscription[subscriptionID] = regions + } + oldState := regions[regionID] + regions[regionID] = state + t.Unlock() + + if oldState == nil { + return + } + log.Warn("region request state overwritten", + zap.Uint64("workerID", t.workerID), + zap.Uint64("subscriptionID", uint64(subscriptionID)), + zap.Uint64("regionID", regionID)) + oldState.abortScanIfNeeded() +} + +// Get returns the tracked region state for events received from TiKV. +// +// A nil return means the event belongs to a stale region, for example after a +// deregister request or stream cleanup. +func (t *regionTracker) Get(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { + t.RLock() + defer t.RUnlock() + + if regions, ok := t.regionsBySubscription[subscriptionID]; ok { + return regions[regionID] + } + return nil +} + +// RemoveRegion removes one tracked region. +func (t *regionTracker) RemoveRegion(subscriptionID SubscriptionID, regionID uint64) *regionFeedState { + t.Lock() + var state *regionFeedState + if regions, ok := t.regionsBySubscription[subscriptionID]; ok { + state = regions[regionID] + delete(regions, regionID) + if len(regions) == 0 { + delete(t.regionsBySubscription, subscriptionID) + } + } + t.Unlock() + return state +} + +// RemoveSubscription removes all tracked regions for a deregistered +// subscription. +func (t *regionTracker) RemoveSubscription(subscriptionID SubscriptionID) []*regionFeedState { + t.Lock() + regions := t.regionsBySubscription[subscriptionID] + delete(t.regionsBySubscription, subscriptionID) + t.Unlock() + + return collectTrackedRegionStates(regions) +} + +// Drain removes all tracked regions when the worker stream is closed. +func (t *regionTracker) Drain() map[SubscriptionID][]*regionFeedState { + t.Lock() + regionsBySubscription := t.regionsBySubscription + t.regionsBySubscription = make(map[SubscriptionID]trackedRegionStates) + t.Unlock() + + statesBySubscription := make(map[SubscriptionID][]*regionFeedState, len(regionsBySubscription)) + for subID, regions := range regionsBySubscription { + statesBySubscription[subID] = collectTrackedRegionStates(regions) + } + return statesBySubscription +} + +func collectTrackedRegionStates(regions trackedRegionStates) []*regionFeedState { + states := make([]*regionFeedState, 0, len(regions)) + for _, state := range regions { + if state == nil { + continue + } + states = append(states, state) + } + return states +} diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go index 05664cf763..9669c21272 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/requested_store.go @@ -90,12 +90,7 @@ type requestedStore struct { notifyEnqueued bool } -func newRequestedStore( - ctx context.Context, - eg *errgroup.Group, - scheduler *regionRequestScheduler, - storeAddr string, -) *requestedStore { +func newRequestedStore(scheduler *regionRequestScheduler, storeAddr string) *requestedStore { pendingRegionRequestQueueSize := scheduler.config.PendingRegionRequestQueueSize regionRequestWorkerPerStore := scheduler.config.RegionRequestWorkerPerStore perWorkerQueueSize := pendingRegionRequestQueueSize / int(regionRequestWorkerPerStore) @@ -115,11 +110,10 @@ func newRequestedStore( } rs.quota = newStoreQuota(perStoreQuotaSize, rs.NotifyAvailable) for range regionRequestWorkerPerStore { + requestCache := newRequestCache(perWorkerQueueSize, rs.NotifyAvailable) requestWorker := newRegionRequestWorker( - ctx, - eg, - rs, - perWorkerQueueSize, + storeAddr, + requestCache, scheduler.upstream, scheduler.eventSink, scheduler.failureHandler, @@ -129,9 +123,17 @@ func newRequestedStore( return rs } +func (rs *requestedStore) Run(ctx context.Context, eg *errgroup.Group) { + for _, worker := range rs.requestWorkers { + eg.Go(func() error { + return worker.Run(ctx) + }) + } +} + func (rs *requestedStore) Close() { for _, worker := range rs.requestWorkers { - worker.ClearRegionRequests() + worker.requestCache.close() } } @@ -143,7 +145,7 @@ func (rs *requestedStore) AddRegion( start := int(rs.nextWorker.Add(1)) % len(workers) for i := range len(workers) { worker := workers[(start+i)%len(workers)] - ok, err := worker.AddRegionRequest(ctx, region, force, quota) + ok, err := worker.requestCache.add(ctx, region, force, quota) if err != nil || ok { return ok, worker, err } diff --git a/logservice/logpuller/subscription_client_test.go b/logservice/logpuller/subscription_client_test.go index a42f586b1d..34dc76170b 100644 --- a/logservice/logpuller/subscription_client_test.go +++ b/logservice/logpuller/subscription_client_test.go @@ -106,11 +106,12 @@ func TestGenerateResolveLockTask(t *testing.T) { worker := ®ionRequestWorker{ requestCache: &requestCache{}, + tracker: newRegionTracker(0), } // Lock another range, no task will be triggered before initialized. res = span.rangeLock.LockRange(context.Background(), []byte{'c'}, []byte{'d'}, 2, 100) require.Equal(t, regionlock.LockRangeStatusSuccess, res.Status) - state := newRegionFeedState(regionInfo{lockedRangeState: res.LockedRangeState, subscribedSpan: span}, 1, worker) + state := newRegionFeedState(regionInfo{lockedRangeState: res.LockedRangeState, subscribedSpan: span}, 1, worker, nil) span.resolveStaleLocks(200) select { case <-client.resolveLockTaskCh: @@ -499,17 +500,17 @@ func TestEnqueueDeregisterToAllStoresUsesControlQueue(t *testing.T) { subscribedSpan: &subscribedSpan{subID: SubscriptionID(2)}, lockedRangeState: ®ionlock.LockedRangeState{}, } - ok, err := worker.AddRegionRequest(ctx, dummyRegion, true, testRegionRequestQuota()) + ok, err := worker.requestCache.add(ctx, dummyRegion, true, testRegionRequestQuota()) require.NoError(t, err) require.True(t, ok) scheduler.BroadcastDeregister(SubscriptionID(1), true) - require.Equal(t, 1, worker.controlQueue.ch.Len()) + require.Equal(t, 1, worker.controlQueue.len()) req, ok := worker.controlQueue.tryPop() require.True(t, ok) require.Equal(t, SubscriptionID(1), req.subID) require.True(t, req.filterLoop) - require.Equal(t, 1, worker.PendingRequestCount()) + require.Equal(t, 1, worker.requestCache.pendingCount()) } func TestRequestedStoreDeferredTasksPriority(t *testing.T) { diff --git a/utils/notifyqueue/notify_queue.go b/utils/notifyqueue/notify_queue.go new file mode 100644 index 0000000000..331aaf4b02 --- /dev/null +++ b/utils/notifyqueue/notify_queue.go @@ -0,0 +1,77 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notifyqueue provides a FIFO queue with a selectable ready signal. +package notifyqueue + +import "github.com/pingcap/ticdc/utils/deque" + +// Queue is a FIFO queue with a wake-up channel. +// +// Queue is not safe for concurrent use. Callers must serialize Push, TryPop, +// Drain, and Len by themselves. +// +// Ready returns a wake-up hint. Receiving from it does not guarantee TryPop +// will return an item, and callers must re-check the queue state after wake-up. +type Queue[T any] struct { + queue *deque.Deque[T] + ready chan struct{} +} + +// New creates an empty Queue. +func New[T any]() *Queue[T] { + return &Queue[T]{ + queue: deque.NewDequeDefault[T](), + ready: make(chan struct{}, 1), + } +} + +// Push appends an item and signals Ready. +func (q *Queue[T]) Push(item T) { + q.queue.PushBack(item) + q.signal() +} + +// TryPop pops one item from the front of the queue. +func (q *Queue[T]) TryPop() (T, bool) { + return q.queue.PopFront() +} + +// Drain removes and returns all queued items in FIFO order. +func (q *Queue[T]) Drain() []T { + items := make([]T, 0, q.queue.Length()) + for { + item, ok := q.queue.PopFront() + if !ok { + return items + } + items = append(items, item) + } +} + +// Len returns the number of queued items. +func (q *Queue[T]) Len() int { + return q.queue.Length() +} + +// Ready returns a wake-up channel for consumers. +func (q *Queue[T]) Ready() <-chan struct{} { + return q.ready +} + +func (q *Queue[T]) signal() { + select { + case q.ready <- struct{}{}: + default: + } +} diff --git a/utils/notifyqueue/notify_queue_test.go b/utils/notifyqueue/notify_queue_test.go new file mode 100644 index 0000000000..1532f23d3d --- /dev/null +++ b/utils/notifyqueue/notify_queue_test.go @@ -0,0 +1,83 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package notifyqueue + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestQueuePushPopAndReady(t *testing.T) { + q := New[int]() + + select { + case <-q.Ready(): + t.Fatal("empty queue should not be ready") + default: + } + + q.Push(1) + q.Push(2) + require.Equal(t, 2, q.Len()) + + select { + case <-q.Ready(): + default: + t.Fatal("push should signal ready") + } + + v, ok := q.TryPop() + require.True(t, ok) + require.Equal(t, 1, v) + v, ok = q.TryPop() + require.True(t, ok) + require.Equal(t, 2, v) + require.Equal(t, 0, q.Len()) + + _, ok = q.TryPop() + require.False(t, ok) +} + +func TestQueueReadyIsCoalesced(t *testing.T) { + q := New[int]() + + q.Push(1) + q.Push(2) + + select { + case <-q.Ready(): + default: + t.Fatal("push should signal ready") + } + + select { + case <-q.Ready(): + t.Fatal("ready signal should be coalesced") + default: + } +} + +func TestQueueDrain(t *testing.T) { + q := New[int]() + q.Push(1) + q.Push(2) + q.Push(3) + + require.Equal(t, []int{1, 2, 3}, q.Drain()) + require.Equal(t, 0, q.Len()) + + _, ok := q.TryPop() + require.False(t, ok) +} From 359be262213df0aa94bd6e4d95e3d925c684cefe Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 1 Jul 2026 13:15:49 -0700 Subject: [PATCH 20/21] add metrics --- logservice/logpuller/region_priority_task.go | 3 + .../logpuller/region_request_scheduler.go | 57 ++++++++++++++----- logservice/logpuller/region_request_worker.go | 5 ++ logservice/logpuller/requested_store.go | 29 +++++++++- pkg/metrics/log_puller.go | 49 ++++++++++++++++ 5 files changed, 128 insertions(+), 15 deletions(-) diff --git a/logservice/logpuller/region_priority_task.go b/logservice/logpuller/region_priority_task.go index b88dba051f..9e954526b1 100644 --- a/logservice/logpuller/region_priority_task.go +++ b/logservice/logpuller/region_priority_task.go @@ -15,6 +15,7 @@ package logpuller import ( "fmt" + "time" "github.com/tikv/client-go/v2/oracle" ) @@ -49,6 +50,7 @@ type regionPriorityTask struct { taskType TaskType priority int seq uint64 + createTime time.Time regionInfo regionInfo heapIndex int // for heap.Item interface } @@ -59,6 +61,7 @@ func newRegionPriorityTask(taskType TaskType, regionInfo regionInfo, currentTs u taskType: taskType, priority: calculateRegionTaskPriority(taskType, regionInfo, currentTs), seq: seq, + createTime: time.Now(), regionInfo: regionInfo, heapIndex: 0, // 0 means not in heap } diff --git a/logservice/logpuller/region_request_scheduler.go b/logservice/logpuller/region_request_scheduler.go index b52065f92b..21c2020fe7 100644 --- a/logservice/logpuller/region_request_scheduler.go +++ b/logservice/logpuller/region_request_scheduler.go @@ -15,8 +15,10 @@ package logpuller import ( "context" + "strconv" "sync" "sync/atomic" + "time" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/metrics" @@ -28,6 +30,12 @@ import ( "golang.org/x/sync/errgroup" ) +const ( + deferReasonStorePending = "store_pending" + deferReasonStoreQuota = "store_quota" + deferReasonWorkerCache = "worker_cache" +) + // regionRequestScheduler owns region request admission from the global // priority queue to per-store request workers. type regionRequestScheduler struct { @@ -144,8 +152,20 @@ func (s *regionRequestScheduler) UpdateMetrics() { count := 0 s.stores.Range(func(_, value any) bool { store := value.(*requestedStore) + quotaUsed, quotaCapacity := store.quota.Snapshot() + metrics.SubscriptionClientStoreDeferredRegionCount.WithLabelValues(store.storeAddr). + Set(float64(store.PendingTaskCount())) + metrics.SubscriptionClientStoreQuotaGauge.WithLabelValues(store.storeAddr, "used"). + Set(float64(quotaUsed)) + metrics.SubscriptionClientStoreQuotaGauge.WithLabelValues(store.storeAddr, "capacity"). + Set(float64(quotaCapacity)) for _, worker := range store.requestWorkers { - count += worker.requestCache.pendingCount() + pendingCount := worker.requestCache.pendingCount() + count += pendingCount + metrics.SubscriptionClientWorkerPendingRegionCount.WithLabelValues( + store.storeAddr, + strconv.FormatUint(worker.workerID, 10), + ).Set(float64(pendingCount)) } return true }) @@ -191,7 +211,7 @@ type getRequestedStoreFunc func(storeAddr string) *requestedStore func (s *regionRequestScheduler) handleDeferredTasks(ctx context.Context, store *requestedStore) error { for { - task, ok := store.pendingTasks.TryPop() + task, ok := store.TryPopPendingTask() if !ok { return nil } @@ -211,12 +231,13 @@ func (s *regionRequestScheduler) handleDeferredTasks(ctx context.Context, store continue } - ok, err := s.tryAdmitTask(ctx, store, task, region) + ok, reason, err := s.tryAdmitTask(ctx, store, task, region) if err != nil { return err } if !ok { - store.pendingTasks.Push(task) + store.PushPendingTask(task) + s.observeDeferredTask(store, reason) return nil } } @@ -234,18 +255,20 @@ func (s *regionRequestScheduler) handleNewTask( task.regionInfo = region store := getStore(region.rpcCtx.Addr) - if store.pendingTasks.Len() > 0 { - store.pendingTasks.Push(task) + if store.PendingTaskCount() > 0 { + store.PushPendingTask(task) store.NotifyAvailable() + s.observeDeferredTask(store, deferReasonStorePending) return nil } - ok, err := s.tryAdmitTask(ctx, store, task, region) + ok, reason, err := s.tryAdmitTask(ctx, store, task, region) if err != nil { return err } if !ok { - store.pendingTasks.Push(task) + store.PushPendingTask(task) + s.observeDeferredTask(store, reason) } return nil } @@ -255,11 +278,11 @@ func (s *regionRequestScheduler) tryAdmitTask( store *requestedStore, task *regionPriorityTask, region regionInfo, -) (bool, error) { +) (bool, string, error) { force := task.Priority() <= forcedPriorityBase acquiredQuota, ok := store.quota.TryAcquire() if !ok { - return false, nil + return false, deferReasonStoreQuota, nil } ok, worker, err := store.AddRegion(ctx, region, force, acquiredQuota) if err != nil { @@ -268,17 +291,25 @@ func (s *regionRequestScheduler) tryAdmitTask( zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.Error(err)) - return false, err + return false, "", err } if !ok { acquiredQuota.Release() - return false, nil + return false, deferReasonWorkerCache, nil } + metrics.SubscriptionClientRegionRequestAdmitDuration.Observe(time.Since(task.createTime).Seconds()) log.Debug("subscription client will request a region", zap.Uint64("workID", worker.workerID), zap.Uint64("subscriptionID", uint64(region.subscribedSpan.subID)), zap.Uint64("regionID", region.verID.GetID()), zap.String("addr", store.storeAddr)) - return true, nil + return true, "", nil +} + +func (s *regionRequestScheduler) observeDeferredTask(store *requestedStore, reason string) { + if reason == "" { + return + } + metrics.SubscriptionClientRegionRequestDeferCounter.WithLabelValues(store.storeAddr, reason).Inc() } diff --git a/logservice/logpuller/region_request_worker.go b/logservice/logpuller/region_request_worker.go index 4720ed2d44..747826a875 100644 --- a/logservice/logpuller/region_request_worker.go +++ b/logservice/logpuller/region_request_worker.go @@ -15,6 +15,7 @@ package logpuller import ( "context" + "strconv" "sync" "sync/atomic" "time" @@ -469,6 +470,10 @@ func (s *regionRequestWorker) processRegionSendTask( state.markStopped(err) return err } + metrics.SubscriptionClientRegionRequestSendCounter.WithLabelValues( + s.storeAddr, + strconv.FormatUint(s.workerID, 10), + ).Inc() } regionReq = nil continue diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go index 9669c21272..cd73630956 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/requested_store.go @@ -69,6 +69,12 @@ func (q *storeQuota) TryAcquire() (*regionRequestQuota, bool) { }, true } +func (q *storeQuota) Snapshot() (used int, capacity int) { + q.mu.Lock() + defer q.mu.Unlock() + return q.used, q.capacity +} + // requestedStore is the scheduler's local state for one TiKV store. type requestedStore struct { scheduler *regionRequestScheduler @@ -83,8 +89,9 @@ type requestedStore struct { // pendingTasks holds tasks that have been routed to this store but are // waiting for store quota or worker request-cache capacity. It is mutated - // only by the single regionRequestScheduler.Run loop. - pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] + // by the scheduler loop and read by metrics updater. + pendingTasksMu sync.Mutex + pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] notifyMu sync.Mutex notifyEnqueued bool @@ -137,6 +144,24 @@ func (rs *requestedStore) Close() { } } +func (rs *requestedStore) PushPendingTask(task *regionPriorityTask) { + rs.pendingTasksMu.Lock() + rs.pendingTasks.Push(task) + rs.pendingTasksMu.Unlock() +} + +func (rs *requestedStore) TryPopPendingTask() (*regionPriorityTask, bool) { + rs.pendingTasksMu.Lock() + defer rs.pendingTasksMu.Unlock() + return rs.pendingTasks.TryPop() +} + +func (rs *requestedStore) PendingTaskCount() int { + rs.pendingTasksMu.Lock() + defer rs.pendingTasksMu.Unlock() + return rs.pendingTasks.Len() +} + func (rs *requestedStore) AddRegion( ctx context.Context, region regionInfo, force bool, quota *regionRequestQuota, ) (bool, *regionRequestWorker, error) { diff --git a/pkg/metrics/log_puller.go b/pkg/metrics/log_puller.go index e874aa4428..7b791249a7 100644 --- a/pkg/metrics/log_puller.go +++ b/pkg/metrics/log_puller.go @@ -80,6 +80,49 @@ var ( Name: "requested_region_count", Help: "The number of requested regions", }, []string{"state"}) + SubscriptionClientWorkerPendingRegionCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "subscription_client", + Name: "worker_pending_region_count", + Help: "The number of pending region requests in each worker", + }, []string{"store", "worker"}) + SubscriptionClientStoreDeferredRegionCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "subscription_client", + Name: "store_deferred_region_count", + Help: "The number of deferred region requests for each store", + }, []string{"store"}) + SubscriptionClientStoreQuotaGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "subscription_client", + Name: "store_region_request_quota", + Help: "The region request quota usage for each store", + }, []string{"store", "type"}) + SubscriptionClientRegionRequestSendCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "subscription_client", + Name: "region_request_send_count", + Help: "The number of region requests sent to TiKV", + }, []string{"store", "worker"}) + SubscriptionClientRegionRequestDeferCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "subscription_client", + Name: "region_request_defer_count", + Help: "The number of region requests deferred by the scheduler", + }, []string{"store", "reason"}) + SubscriptionClientRegionRequestAdmitDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "subscription_client", + Name: "region_request_admit_duration", + Help: "duration (s) from region request task creation to admission into a worker", + Buckets: prometheus.ExponentialBuckets(0.00004, 2.0, 28), // 40us to 1.5h + }) RegionRequestFinishScanDuration = prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: "ticdc", @@ -165,6 +208,12 @@ func initLogPullerMetrics(registry *prometheus.Registry) { registry.MustRegister(LogPullerMatcherCount) registry.MustRegister(LogPullerResolvedTsLag) registry.MustRegister(SubscriptionClientRequestedRegionCount) + registry.MustRegister(SubscriptionClientWorkerPendingRegionCount) + registry.MustRegister(SubscriptionClientStoreDeferredRegionCount) + registry.MustRegister(SubscriptionClientStoreQuotaGauge) + registry.MustRegister(SubscriptionClientRegionRequestSendCounter) + registry.MustRegister(SubscriptionClientRegionRequestDeferCounter) + registry.MustRegister(SubscriptionClientRegionRequestAdmitDuration) registry.MustRegister(SubscriptionClientAddRegionRequestDuration) registry.MustRegister(RegionRequestFinishScanDuration) registry.MustRegister(SubscriptionClientSubscribedRegionCount) From 3cc1f1a4312667698d3c91bf31f7967fc927c983 Mon Sep 17 00:00:00 2001 From: lidezhu Date: Wed, 1 Jul 2026 13:22:33 -0700 Subject: [PATCH 21/21] improve metrics --- logservice/logpuller/requested_store.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/logservice/logpuller/requested_store.go b/logservice/logpuller/requested_store.go index cd73630956..11a47a4448 100644 --- a/logservice/logpuller/requested_store.go +++ b/logservice/logpuller/requested_store.go @@ -89,9 +89,12 @@ type requestedStore struct { // pendingTasks holds tasks that have been routed to this store but are // waiting for store quota or worker request-cache capacity. It is mutated - // by the scheduler loop and read by metrics updater. - pendingTasksMu sync.Mutex - pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] + // only by the single regionRequestScheduler.Run loop. + pendingTasks *priorityqueue.PriorityQueue[*regionPriorityTask] + // pendingTaskCount mirrors pendingTasks.Len() for metrics. It lets the + // metrics updater observe the size without touching the scheduler-owned + // priority queue. + pendingTaskCount atomic.Int64 notifyMu sync.Mutex notifyEnqueued bool @@ -145,21 +148,20 @@ func (rs *requestedStore) Close() { } func (rs *requestedStore) PushPendingTask(task *regionPriorityTask) { - rs.pendingTasksMu.Lock() rs.pendingTasks.Push(task) - rs.pendingTasksMu.Unlock() + rs.pendingTaskCount.Add(1) } func (rs *requestedStore) TryPopPendingTask() (*regionPriorityTask, bool) { - rs.pendingTasksMu.Lock() - defer rs.pendingTasksMu.Unlock() - return rs.pendingTasks.TryPop() + task, ok := rs.pendingTasks.TryPop() + if ok { + rs.pendingTaskCount.Add(-1) + } + return task, ok } func (rs *requestedStore) PendingTaskCount() int { - rs.pendingTasksMu.Lock() - defer rs.pendingTasksMu.Unlock() - return rs.pendingTasks.Len() + return int(rs.pendingTaskCount.Load()) } func (rs *requestedStore) AddRegion(