pq: deadlock detected

[dtornow]

The resonate server logs the following errors

time=2025-09-24T15:37:28.154+07:00 level=ERROR msg="failed to claim task" req="ClaimTask(id=__invoke:d21852ea-029a-49d2-afc1-3d6be4f5f546, counter=1, processId=63a7bc3223664ebfad1391c80e30d18f, ttl=10000)" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:28.154+07:00 level=ERROR msg="failed to claim task" req="ClaimTask(id=__invoke:1841a97f-d1d3-4732-a4f2-6cefaa21709f, counter=1, processId=9b2628d8bbd4492892b1dd003df0e130, ttl=10000)" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed store execution" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed store execution" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed store execution" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed store execution" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed store execution" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed to read promise" req="CompletePromise(id=d539bfc8-d962-46e8-a334-eaecef82ae2e, idempotencyKey=d539bfc8-d962-46e8-a334-eaecef82ae2e, strict=false, state=RESOLVED, value=Value(headers=map[resonate:format-py:\"d539bfc8-d962-46e8-a334-eaecef82ae2e\"], data=\"d539bfc8-d962-46e8-a334-eaecef82ae2e\"))" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed to claim task" req="ClaimTask(id=__invoke:293de9d2-c766-4508-87f0-b54a2e4052b9, counter=1, processId=85853584b21e40e09b2538ffb6fbec3a, ttl=10000)" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed to claim task" req="ClaimTask(id=__invoke:22d8a3b7-f8de-4a1a-98ed-acf3743f0373, counter=1, processId=2a64d19d54284c6a9cd1aa030028fb09, ttl=10000)" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed to claim task" req="ClaimTask(id=__invoke:02b43412-38a2-4dc0-8aa6-9e54d8a02929, counter=1, processId=56143969f0034f76b34c99199811587c, ttl=10000)" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

time=2025-09-24T15:37:29.240+07:00 level=ERROR msg="failed to read promise" req="CompletePromise(id=75dfde53-69c3-44bc-b499-5f7386614bc4, idempotencyKey=75dfde53-69c3-44bc-b499-5f7386614bc4, strict=false, state=RESOLVED, value=Value(headers=map[resonate:format-py:\"75dfde53-69c3-44bc-b499-5f7386614bc4\"], data=\"75dfde53-69c3-44bc-b499-5f7386614bc4\"))" err="pq: deadlock detected at github.com/resonatehq/resonate/internal/app/subsystems/aio/store/postgres.(*PostgresStoreWorker).updateTask /home/kahnwong/go/pkg/mod/github.com/resonatehq/resonate@v0.7.15/internal/app/subsystems/aio/store/postgres/postgres.go:1607"

[dtornow]

Thank you to @kahnwong for performing a load test and reporting the error

[Tomperez98]

Some findings regarding the issue.

• It doesn't seems to cause any correctness problem on the application, neither cause the server to crash.
• Seems to be derived when using more than one Postgres workers --aio-store-postgres-workers 2

// internal/app/subsystems/aio/store/postgres/postgres.go

func New(aio aio.AIO, metrics *metrics.Metrics, config *Config) (*PostgresStore, error) {
	sq := make(chan *bus.SQE[t_aio.Submission, t_aio.Completion], config.Size)
	workers := make([]*PostgresStoreWorker, config.Workers)

	rawQuery := make([]string, len(config.Query))
	for i, q := range util.OrderedRangeKV(config.Query) {
		rawQuery[i] = fmt.Sprintf("%s=%s", q.Key, q.Value)
	}

	dbUrl := &url.URL{
		User:     url.UserPassword(config.Username, config.Password),
		Host:     fmt.Sprintf("%s:%s", config.Host, config.Port),
		Path:     config.Database,
		Scheme:   "postgres",
		RawQuery: strings.Join(rawQuery, "&"),
	}

	db, err := sql.Open("postgres", dbUrl.String())
	if err != nil {
		return nil, err
	}

	db.SetMaxOpenConns(config.Workers)
	db.SetMaxIdleConns(config.Workers)
	db.SetConnMaxIdleTime(0)

	for i := 0; i < config.Workers; i++ {
		workers[i] = &PostgresStoreWorker{
			config:  config,
			i:       i,
			db:      db,
			sq:      sq,
			flush:   make(chan int64, 1),
			aio:     aio,
			metrics: metrics,
		}
	}

	return &PostgresStore{
		config:  config,
		sq:      sq,
		db:      db,
		workers: workers,
	}, nil
}

[flossypurse]

recommended action:

• you can safely ignore this error
• you can scale down your postres workers to 1 (which has minimal impact on perf)