From 8767c4940ecc9ba1386e4dbe156575d1548adce7 Mon Sep 17 00:00:00 2001 From: barry Date: Tue, 16 Jun 2026 23:46:20 +0800 Subject: [PATCH 1/7] refactor(cloudevent): migrate catdogs runtime, proto, and codegen Adopt jobManager/interceptor architecture, split cloudeventoption extensions, and align protoc-gen-go-cloudevent2 output with catdogs conventions. Co-authored-by: Cursor --- .../protoc-gen-go-cloudevent2/internal/gen.go | 87 ++--- .../internal/version.go | 2 +- component/cloudevent/_doc.go | 11 + component/cloudevent/aaa.go | 41 +- component/cloudevent/client.go | 349 +++--------------- component/cloudevent/config.go | 79 ++-- component/cloudevent/context.go | 115 +++--- component/cloudevent/errors.go | 25 +- component/cloudevent/publisher.go | 191 ++++------ component/cloudevent/register.go | 52 ++- component/cloudevent/subscriber.go | 252 +++++++++++++ component/cloudevent/util.go | 46 ++- proto/cloudevent/options.pb.go | 124 ------- proto/cloudevent/types.pb.go | 83 +++-- proto/cloudevent/types.proto | 45 +-- proto/cloudeventoption/options.pb.go | 125 +++++++ .../options.proto | 8 +- proto/cloudeventoption/protobuf.plugin.yaml | 2 + protobuf.yaml | 5 +- 19 files changed, 813 insertions(+), 829 deletions(-) create mode 100644 component/cloudevent/_doc.go create mode 100644 component/cloudevent/subscriber.go delete mode 100644 proto/cloudevent/options.pb.go create mode 100644 proto/cloudeventoption/options.pb.go rename proto/{cloudevent => cloudeventoption}/options.proto (54%) create mode 100644 proto/cloudeventoption/protobuf.plugin.yaml diff --git a/cmds/protoc-gen-go-cloudevent2/internal/gen.go b/cmds/protoc-gen-go-cloudevent2/internal/gen.go index b0969ea5..e63aed34 100644 --- a/cmds/protoc-gen-go-cloudevent2/internal/gen.go +++ b/cmds/protoc-gen-go-cloudevent2/internal/gen.go @@ -13,18 +13,22 @@ import ( "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "github.com/pubgo/funk/v2" "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/component/cloudevent" cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + cloudeventoptionpb "github.com/pubgo/funk/v2/proto/cloudeventoption" + "github.com/pubgo/funk/v2/result" "github.com/pubgo/funk/v2/stack" ) var ( - cloudeventPkg = reflect.TypeOf(cloudevent.Client{}).PkgPath() - jobTypesPkg = reflect.TypeOf(cloudeventpb.PushEventOptions{}).PkgPath() - ctxPkg = stack.CallerWithFunc(context.WithTimeout).Pkg - assertPkt = stack.CallerWithFunc(assert.Assert).Pkg - protojsonPkt = stack.CallerWithFunc(protojson.Marshal).Pkg + cloudeventPkg = reflect.TypeOf(cloudevent.Client{}).PkgPath() + resultTypesPkg = stack.CallerWithFunc(result.Recovery).Pkg + ctxPkg = stack.CallerWithFunc(context.WithTimeout).Pkg + assertPkt = stack.CallerWithFunc(assert.Assert).Pkg + protojsonPkt = stack.CallerWithFunc(protojson.Marshal).Pkg + funkPkg = stack.CallerWithFunc(funk.IsNil).Pkg ) type eventInfo struct { @@ -40,7 +44,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated filename := file.GeneratedFilenamePrefix + ".cloudevent.pb.go" genFile := jen.NewFile(string(file.GoPackageName)) genFile.HeaderComment("Code generated by protoc-gen-go-cloudevent. DO NOT EDIT.") - genFile.HeaderComment("versions:") + genFile.HeaderComment("Versions:") genFile.HeaderComment(fmt.Sprintf(" - protoc-gen-go-cloudevent %s", Version)) genFile.HeaderComment(fmt.Sprintf(" - protoc %s", protocVersion(gen))) if file.Proto.GetOptions().GetDeprecated() { @@ -58,7 +62,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated events := make(map[string]map[string]*eventInfo) for _, srv := range file.Services { - job, ok := proto.GetExtension(srv.Desc.Options(), cloudeventpb.E_Job).(*cloudeventpb.CloudEventServiceOptions) + job, ok := proto.GetExtension(srv.Desc.Options(), cloudeventoptionpb.E_Job).(*cloudeventpb.CloudEventServiceOptions) if !ok || job == nil { continue } @@ -70,7 +74,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated } for _, m := range srv.Methods { - jobSubject, ok := proto.GetExtension(m.Desc.Options(), cloudeventpb.E_Subject).(*cloudeventpb.CloudEventMethodOptions) + jobSubject, ok := proto.GetExtension(m.Desc.Options(), cloudeventoptionpb.E_Subject).(*cloudeventpb.CloudEventMethodOptions) if !ok || jobSubject == nil { continue } @@ -109,7 +113,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated srvInfo := getSrv(subjects) jobKeyPrefix := strings.ReplaceAll(srvInfo.GoName, "InnerService", "") jobKeyPrefix = strings.ReplaceAll(jobKeyPrefix, "Inner", "") - jobKeyName := fmt.Sprintf("%sCloudEventKey", jobKeyPrefix) + jobKeyName := fmt.Sprintf("%sCloudEventJobKey", jobKeyPrefix) genFile.Const(). Id(jobKeyName). Op("="). @@ -120,7 +124,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated for _, subName := range subjectNames { info := subjects[subName] - keyName := fmt.Sprintf("%sCloudEventKey", info.mth.GoName) + keyName := fmt.Sprintf("%sCloudEventSubjectKey", info.mth.GoName) genFile.Commentf("%s /%s/%s", keyName, info.srv.Desc.FullName(), info.mth.GoName) genFile.Comment(strings.TrimSpace(info.mth.Comments.Leading.String())) genFile.Const(). @@ -129,25 +133,6 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated Lit(subName) } - //for _, subName := range subjectNames { - // //info := subjects[subName] - // //var dd = string(assert.Must1(protojson.Marshal(info.subject))) - // //var keyName = fmt.Sprintf("%sCloudEventKey", info.mth.GoName) - // //genFile.Var().Id("_").Op("="). - // // Qual(cloudeventPkg, "RegisterSubject"). - // // Call( - // // jen.Id(keyName), - // // jen.Lit(fmt.Sprintf("/%s/%s", info.srv.Desc.FullName(), info.mth.GoName)), - // // jen.Func().Params().Params(jen.Op("*").Qual(jobTypesPkg, "CloudEventSubject")). - // // BlockFunc(func(group *jen.Group) { - // // group.Var().Id("data=[]byte").Call(jen.Lit(fmt.Sprintf(`%s`, dd))) - // // group.Var().Id("p").Qual(jobTypesPkg, "CloudEventSubject") - // // group.Qual(assertPkt, "Must").Call(jen.Qual(protojsonPkt, "Unmarshal").Call(jen.Id("data, &p"))) - // // group.Return().Id("&p") - // // }).Call(), - // // ).Line() - //} - subjectValues := lo.Values(subjects) cloudEventName := fmt.Sprintf("%sCloudEvent", subjectValues[0].srv.GoName) genFile.Type().Id(cloudEventName).StructFunc(func(group *jen.Group) { @@ -164,17 +149,16 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated Params( jen.Id("jobCli").Op("*").Qual(cloudeventPkg, "Client"), jen.Id("event").Id(cloudEventName), - jen.Id("opts").Op("...").Op("*").Qual(jobTypesPkg, "RegisterJobOptions"), + jen.Id("opts").Op("...").Qual(cloudeventPkg, "RegisterOpt"), ).BlockFunc(func(group *jen.Group) { for _, ss := range subjectValues { - keyName := fmt.Sprintf("%sCloudEventKey", ss.mth.GoName) + keyName := fmt.Sprintf("%sCloudEventSubjectKey", ss.mth.GoName) group.If(jen.Id("event").Dot("On" + ss.mth.GoName)).Op("!=").Nil().BlockFunc(func(group *jen.Group) { group.Qual(cloudeventPkg, "RegisterJobHandler").Call( jen.Id("jobCli"), jen.Id(jobKeyName), jen.Id(keyName), jen.Id("event").Dot("On"+ss.mth.GoName), - // jen.Qual(cloudeventPkg, "WrapHandler").Call(jen.Id("event").Dot(ss.mth.GoName)), jen.Id("opts").Op("..."), ) }).Line() @@ -184,25 +168,30 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated publisher := fmt.Sprintf("%sPublisher", cloudEventName) genFile.Type().Id(publisher).StructFunc(func(group *jen.Group) { group.Id("Client").Op("*").Qual(cloudeventPkg, "Client") + group.Id("Opt").Qual(cloudeventPkg, "PubOpt") + group.Id("Interceptors").Op("[]").Qual(cloudeventPkg, "PubInterceptor") }) for _, ss := range subjectValues { - mthName := fmt.Sprintf("Push%sEvent", ss.mth.GoName) - keyName := fmt.Sprintf("%sCloudEventKey", ss.mth.GoName) + mthName := handlerPushEventName(ss.mth.GoName) + keyName := fmt.Sprintf("%sCloudEventSubjectKey", ss.mth.GoName) genFile.Func(). Params(jen.Id(fmt.Sprintf("a %s", publisher))). Id(mthName). Params( jen.Id("ctx").Qual("context", "Context"), - jen.Id("req").Op("*").Id(ss.mth.Input.GoIdent.GoName), - jen.Id("opts").Op("...").Op("*").Qual(jobTypesPkg, "PushEventOptions"), + jen.Id("req *").Add(getPkg(file, ss.mth.Input.GoIdent)), + jen.Id("opts ...").Qual(cloudeventPkg, "PubOpt"), ). - Params(jen.Op("*").Qual(cloudeventPkg, "PubAckInfo"), jen.Error()). - Block(jen.Return().Id("a.Client").Dot("Publish").Call( - jen.Id("ctx"), - jen.Id(keyName), - jen.Id("req"), - jen.Id("opts").Op("..."), - )) + Params(jen.Qual(resultTypesPkg, "Result").Id("[").Op("*").Qual(cloudeventPkg, "PubAckInfo").Id("]")). + BlockFunc(func(group *jen.Group) { + group.Return().Id("a.Client.Publish").Call( + jen.Id("ctx"), + jen.Id(keyName), + jen.Id("req"), + jen.Id("a.Interceptors"), + jen.Qual(funkPkg, "AppendOf").Id("(a.Opt, opts...)..."), + ) + }) } } @@ -210,11 +199,15 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated return g } -func handlerPushEventName(name string, prefix string) string { - if strings.HasPrefix(name, prefix) { - return name +func handlerPushEventName(name string) string { + if !strings.HasPrefix(name, "Push") { + name = fmt.Sprintf("Push%s", name) + } + + if !strings.HasSuffix(name, "Event") { + name = fmt.Sprintf("%sEvent", name) } - return fmt.Sprintf("%s%s", prefix, name) + return name } func getSrv(data map[string]*eventInfo) *protogen.Service { diff --git a/cmds/protoc-gen-go-cloudevent2/internal/version.go b/cmds/protoc-gen-go-cloudevent2/internal/version.go index 080e6253..c760f98a 100644 --- a/cmds/protoc-gen-go-cloudevent2/internal/version.go +++ b/cmds/protoc-gen-go-cloudevent2/internal/version.go @@ -6,7 +6,7 @@ import ( "google.golang.org/protobuf/compiler/protogen" ) -const Version = "v0.0.4" +const Version = "v0.0.5" func protocVersion(gen *protogen.Plugin) string { v := gen.Request.GetCompilerVersion() diff --git a/component/cloudevent/_doc.go b/component/cloudevent/_doc.go new file mode 100644 index 00000000..326cb350 --- /dev/null +++ b/component/cloudevent/_doc.go @@ -0,0 +1,11 @@ +package cloudevent + +// Promise/Future, Group/Yield iterators, and Go helpers for async work. +// +// Protobuf extensions live in github.com/pubgo/funk/v2/proto/cloudeventoption. +// Message types live in github.com/pubgo/funk/v2/proto/cloudevent. +// +// Iterator.Await drains the value channel to completion before returning the +// first error, so callers never leave Group workers blocked on send. +// +// https://github.com/octu0/chanque diff --git a/component/cloudevent/aaa.go b/component/cloudevent/aaa.go index b4416600..af935e2d 100644 --- a/component/cloudevent/aaa.go +++ b/component/cloudevent/aaa.go @@ -5,27 +5,39 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + "github.com/pubgo/funk/v2/log" + "github.com/pubgo/funk/v2/result" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" - - "github.com/pubgo/funk/v2/log" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" ) var logger = log.GetLogger("cloudevent") -type EventRegister interface { +type Register interface { RegisterCloudEvent(jobCli *Client) } -type ( - EventHandler[T proto.Message] func(ctx context.Context, args T) error - RpcEventHandler[T proto.Message] func(ctx context.Context, args T) (*emptypb.Empty, error) -) +type RegisterJobOptions struct { + Opts *cloudeventpb.RegisterJobOptions + Interceptors []SubInterceptor +} + +type SubInterceptFunc func(ctx context.Context, args proto.Message, handler func(ctx context.Context, args proto.Message) error) error +type SubInterceptor func(next SubInterceptFunc) SubInterceptFunc -type Options = cloudeventpb.PushEventOptions +type RegisterOpt func(opts *RegisterJobOptions) -type PushEventOpt func(opts *Options) +type Handler[T proto.Message] func(ctx context.Context, args T) error +type RpcHandler[T proto.Message] func(ctx context.Context, args T) (*emptypb.Empty, error) + +type PubOpt func(opts *PubOptions) + +type PubInterceptFunc func(ctx context.Context, topic string, args proto.Message, opts *PubOptions, handler func(ctx context.Context, topic string, args proto.Message, opts *PubOptions) result.Result[*PubAckInfo]) result.Result[*PubAckInfo] + +type PubInterceptor func(next PubInterceptFunc) PubInterceptFunc + +type PubOptions = cloudeventpb.PushEventOptions type Consumer struct { jetstream.Consumer @@ -37,3 +49,12 @@ type PubAckInfo struct { Header nats.Header MsgId string } + +type jobManager struct { + managers map[string]*handlerManager + interceptors []SubInterceptor +} + +type handlerManager struct { + handler Handler[proto.Message] +} diff --git a/component/cloudevent/client.go b/component/cloudevent/client.go index f0f0d04f..d5d83da6 100644 --- a/component/cloudevent/client.go +++ b/component/cloudevent/client.go @@ -3,30 +3,23 @@ package cloudevent import ( "context" "fmt" - "net/http" "strings" "time" mapset "github.com/deckarep/golang-set/v2" "github.com/nats-io/nats.go/jetstream" - ants "github.com/panjf2000/ants/v2" - "github.com/rs/zerolog" - "github.com/samber/lo" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" - "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/buildinfo/version" "github.com/pubgo/funk/v2/component/lifecycle" "github.com/pubgo/funk/v2/component/natsclient" "github.com/pubgo/funk/v2/errors" - "github.com/pubgo/funk/v2/log" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" "github.com/pubgo/funk/v2/result" "github.com/pubgo/funk/v2/running" "github.com/pubgo/funk/v2/stack" - "github.com/pubgo/funk/v2/try" "github.com/pubgo/funk/v2/typex" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + "github.com/rs/zerolog" + "github.com/samber/lo" ) type Params struct { @@ -38,14 +31,14 @@ type Params struct { func New(p Params) *Client { js := assert.Must1(jetstream.New(p.Nc.Conn)) return &Client{ - p: p, - js: js, - prefix: DefaultPrefix, - handlers: make(map[string]map[string]EventHandler[proto.Message]), - streams: make(map[string]jetstream.Stream), - consumers: make(map[string]map[string]*Consumer), - jobs: make(map[string]map[string]map[string]*jobEventHandler), - subjects: getAllSubject(), + p: p, + js: js, + prefix: DefaultPrefix, + jobManagers: make(map[string]*jobManager), + streams: make(map[string]jetstream.Stream), + consumers: make(map[string]map[string]*Consumer), + jobs: make(map[string]map[string]map[string]*jobEventHandler), + subjects: getAllSubject(), } } @@ -53,27 +46,16 @@ type Client struct { p Params js jetstream.JetStream - // stream manager - streams map[string]jetstream.Stream - - // jobs: stream->consumer->Consumer + streams map[string]jetstream.Stream consumers map[string]map[string]*Consumer - - // handlers: job name -> subject -> job handler - handlers map[string]map[string]EventHandler[proto.Message] - - // jobs: stream->consumer->subject->jobEventHandler - jobs map[string]map[string]map[string]*jobEventHandler - - // stream, consumer, subject prefix, default: DefaultPrefix - prefix string - - // subjects operation => subject info - subjects map[string]*cloudeventpb.CloudEventMethodOptions + jobManagers map[string]*jobManager + jobs map[string]map[string]map[string]*jobEventHandler + prefix string + subjects map[string]*cloudeventpb.CloudEventMethodOptions } -func (c *Client) initStream() (r error) { - defer result.RecoveryErr(&r) +func (c *Client) initStream() (r result.Error) { + defer result.Recovery(&r) ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout) defer cancel() @@ -82,32 +64,31 @@ func (c *Client) initStream() (r error) { assert.If(c.streams[streamName] != nil, "stream %s already exists", streamName) - // add subject prefix streamSubjects := lo.Map(cfg.Subjects, func(item string, index int) string { return c.subjectName(item) }) metadata := map[string]string{"creator": fmt.Sprintf("%s/%s/%s", version.Project(), version.Version(), running.InstanceID)} - storageType := getStorageType(cfg.Storage) streamCfg := jetstream.StreamConfig{ - Name: streamName, - Subjects: streamSubjects, - Metadata: metadata, - Storage: storageType, - // Retention: jetstream.InterestPolicy, - - // Duplicates is the window within which to track duplicate messages. - // If not set, server default is 2 minutes. + Name: streamName, + Subjects: streamSubjects, + Metadata: metadata, + Storage: getStorageType(cfg.Storage), Duplicates: time.Minute * 5, } - stream := result.Wrap(c.js.CreateOrUpdateStream(ctx, streamCfg)).UnwrapOrLog(func(e result.Event) { - e.Msgf("failed to create stream:%s", streamName) - }) + stream := result.Wrap(c.js.CreateOrUpdateStream(ctx, streamCfg)). + MapErr(func(err error) error { + return errors.Wrapf(err, "failed to create stream:%s", streamName) + }). + UnwrapOrThrow(&r) + if r.IsErr() { + return + } c.streams[streamName] = stream } - return r + return } -func (c *Client) initConsumer() (r error) { - defer result.RecoveryErr(&r) +func (c *Client) initConsumer() (r result.Error) { + defer result.Recovery(&r) allEventKeysSet := mapset.NewSet(lo.MapToSlice(c.subjects, func(key string, value *cloudeventpb.CloudEventMethodOptions) string { return c.subjectName(key) })...) @@ -115,11 +96,10 @@ func (c *Client) initConsumer() (r error) { defer cancel() for jobOrConsumerName, consumers := range c.p.Cfg.Consumers { jobName := jobOrConsumerName - assert.If(c.handlers[jobName] == nil, "failed to find job handler: %s, please impl RegisterCloudJob", jobName) + assert.If(c.jobManagers[jobName] == nil, "failed to find job manager: %s, please impl RegisterCloudJob", jobName) consumerName := jobOrConsumerName for _, cfg := range consumers { - // check subject exists for _, sub := range cfg.Subjects { name := c.subjectName(lo.FromPtr(sub.Name)) assert.If(!allEventKeysSet.Contains(name), "subject:%s not found, please check protobuf define and service", name) @@ -128,12 +108,10 @@ func (c *Client) initConsumer() (r error) { consumerName = c.consumerName(lo.Ternary(cfg.Consumer != nil, lo.FromPtr(cfg.Consumer), consumerName)) streamName := c.streamName(cfg.Stream) - // consumer init typex.DoBlock(func() { if c.consumers[streamName] == nil { c.consumers[streamName] = make(map[string]*Consumer) } - // A streaming consumer can only have one corresponding job handler assert.If(c.consumers[streamName][consumerName] != nil, "consumer %s already exists", consumerName) metadata := map[string]string{"version": fmt.Sprintf("%s/%s", version.Project(), version.Version())} @@ -141,6 +119,7 @@ func (c *Client) initConsumer() (r error) { Name: consumerName, Durable: consumerName, Metadata: metadata, + AckWait: time.Minute * 5, } consumer, err := c.js.CreateOrUpdateConsumer(ctx, streamName, consumerCfg) @@ -172,262 +151,34 @@ func (c *Client) initConsumer() (r error) { }) for subName, subCfg := range subjectMap { - assert.If(c.handlers[jobName][subName] == nil, "job handler not found, job_name=%s sub_name=%s", jobName, subName) + assert.If(c.jobManagers[jobName].managers[subName] == nil, "job manager not found, job=%s subject=%s", jobName, subName) job := &jobEventHandler{ - name: jobName, - handler: c.handlers[jobName][subName], - cfg: subCfg, + name: jobName, + manager: c.jobManagers[jobName].managers[subName], + cfg: subCfg, + interceptors: c.jobManagers[jobName].interceptors, } logger.Info().Func(func(e *zerolog.Event) { e.Str("job_name", job.name) - e.Str("job_handler", stack.CallerWithFunc(job.handler).String()) + e.Str("job_handler", stack.CallerWithFunc(job.manager.handler).String()) e.Any("job_config", subCfg) e.Any("stream_name", streamName) e.Any("consumer_name", consumerName) e.Any("job_subject", subName) - e.Msg("register cloud job handler executor") + e.Msg("register cloud job manager executor") }) c.jobs[streamName][consumerName][subName] = job } }) } } - return r -} - -func (c *Client) doConsumeHandler(streamName, consumerName string, jobSubjects map[string]*jobEventHandler, concurrent int) func(msg jetstream.Msg) { - handler := func(msg jetstream.Msg) { - now := time.Now() - addMsgInfo := func(e *zerolog.Event) { - e.Str("stream", streamName) - e.Str("consumer", consumerName) - e.Any("header", msg.Headers()) - e.Any("msg_id", msg.Headers().Get(jetstream.MsgIDHeader)) - e.Str("subject", msg.Subject()) - e.Str("msg_received_time", now.String()) - e.Str("job_cost", time.Since(now).String()) - } - - logger.Debug().Func(func(e *zerolog.Event) { - addMsgInfo(e) - e.Msg("received cloud job event") - }) - - handlerDelayJob := func() (r result.Result[bool]) { - delayDur := strings.TrimSpace(msg.Headers().Get(DefaultCloudEventDelayKey)) - if delayDur == "" { - return r.WithValue(false) - } - - dur := decodeDelayTime(delayDur). - MapErr(func(err error) error { - return errors.Wrap(err, "failed to parse cloud job delay time") - }). - UnwrapOrThrow(&r) - if r.IsErr() { - return r - } - - // ignore negative delay - if dur < 0 { - return r.WithValue(false) - } - - r = r.WithErr(msg.NakWithDelay(dur)) - if r.IsErr() { - return r - } - - return r.WithValue(true) - } - - if ok, err := handlerDelayJob().UnwrapErr(); err != nil { - logger.Err(err).Func(addMsgInfo).Msg("failed to handle cloud delay job and no ack") - return - } else if ok { - logger.Info().Func(addMsgInfo).Msg("redeliver the message after the given delay") - return - } - - handler := jobSubjects[msg.Subject()] - if handler == nil { - logger.Error().Func(addMsgInfo).Msg("failed to find subject job handler") - return - } - - meta, err := msg.Metadata() - if err != nil { - // no ack, retry always, unless it can recognize special error information - logger.Err(err).Func(addMsgInfo).Msg("failed to parse nats stream msg metadata") - return - } - - cfg := handler.cfg - checkErrAndLog := func(err error, msg string) { - if err == nil { - return - } - - logger.Err(err). - Str("fn_caller", stack.Caller(1).String()). - Func(addMsgInfo). - Any("metadata", meta). - Any("config", cfg). - Any("msg_received_time", now.String()). - Str("job_cost", time.Since(now).String()). - Msg(msg) - } - - err = try.Try(func() error { return c.doHandler(meta, msg, handler, cfg).GetErr() }) - if err == nil { - checkErrAndLog(msg.Ack(), "failed to do msg ack with handler ok") - return - } - - // reject job msg - if isRejectErr(err) { - checkErrAndLog(msg.TermWithReason("reject by caller"), "failed to do msg ack with reject err") - return - } - - backoff := lo.FromPtr(cfg.RetryBackoff) - maxRetries := lo.FromPtr(cfg.MaxRetry) - - // If the error is a redelivery error, then the backoff duration is the error duration - if err1 := isRedeliveryErr(err); err1 != nil { - backoff = err1.delay - } - - // Proactively retry and did not reach the maximum retry count - if meta.NumDelivered < uint64(maxRetries) { - logger.Warn(). - Err(err). - Func(addMsgInfo). - Any("metadata", meta). - Msg("retry nats stream cloud job event") - checkErrAndLog(msg.NakWithDelay(backoff), "failed to retry msg with delay nak") - return - } - - checkErrAndLog(err, "failed to do handler cloud job") - checkErrAndLog(msg.Ack(), "failed to do msg ack with handler error") - } - - pool := assert.Must1(ants.NewPool( - concurrent, - ants.WithLogger(log.NewStd(logger)), - ants.WithNonblocking(false), - )) - // pool.Release() - return func(msg jetstream.Msg) { - if pool.Running() == concurrent { - logger.Warn().Func(func(e *zerolog.Event) { - e.Int("concurrent", concurrent) - e.Str("stream", streamName) - e.Str("consumer", consumerName) - e.Msg("concurrent limit occurred, please check the concurrent limit") - }) - } - if err := pool.Submit(func() { handler(msg) }); err != nil { - logger.Err(err).Func(func(e *zerolog.Event) { - e.Str("stream", streamName) - e.Str("consumer", consumerName) - e.Msg("failed to submit job to pool") - }) - } - } -} - -func (c *Client) doErrHandler(streamName, consumerName string) jetstream.PullConsumeOpt { - return jetstream.ConsumeErrHandler(func(consumeCtx jetstream.ConsumeContext, err error) { - logger.Err(err). - Str("stream", streamName). - Str("consumer", consumerName). - Msg("nats consumer error") - }) -} - -func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { - timeout := lo.FromPtr(cfg.Timeout) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - ctx = log.UpdateFieldsCtx(ctx, log.Fields{ - "sub_subject": msg.Subject(), - "sub_stream": meta.Stream, - "sub_consumer": meta.Consumer, - "sub_msg_id": msg.Headers().Get(jetstream.MsgIDHeader), - "sub_msg_" + DefaultSenderKey: msg.Headers().Get(DefaultSenderKey), - }) - - msgCtx := &Context{ - Header: http.Header(msg.Headers()), - NumDelivered: meta.NumDelivered, - NumPending: meta.NumPending, - Timestamp: meta.Timestamp, - Stream: meta.Stream, - Consumer: meta.Consumer, - Subject: msg.Subject(), - Config: cfg, - } - - now := time.Now() - var args any - defer func() { - if gErr.IsOK() { - return - } - - logger.Err(gErr.GetErr()).Func(func(e *zerolog.Event) { - e.Any("context", msgCtx) - e.Any("args", args) - e.Str("timeout", timeout.String()) - e.Str("start_time", now.String()) - e.Str("job_cost", time.Since(now).String()) - e.Msg("failed to do cloud job handler") - }) - }() - - var pb anypb.Any - err := result.ErrOf(proto.Unmarshal(msg.Data(), &pb)). - MapErr(func(err error) error { - return errors.WrapTags(err, errors.Tags{ - "msg": "failed to unmarshal stream msg data to any proto", - "args": string(msg.Data()), - }) - }) - if err.Throw(&gErr) { - return gErr - } - args = &pb - - dst := result.Wrap(anypb.UnmarshalNew(args.(*anypb.Any), proto.UnmarshalOptions{})). - MapErr(func(err error) error { - return errors.WrapTags(err, errors.Tags{ - "msg": "failed to unmarshal any proto to proto msg", - "args": args, - }) - }) - if dst.Throw(&gErr) { - return gErr - } - - ctx = createCtxWithContext(ctx, msgCtx) - err = result.ErrOf(job.handler(ctx, dst.Unwrap())). - MapErr(func(err error) error { - return errors.WrapTags(err, errors.Tags{ - "msg": "failed to do cloud job handler", - "args": dst, - "any_pb": dst, - }) - }) - return err + return } -func (c *Client) doConsume() (r error) { - defer result.RecoveryErr(&r) +func (c *Client) doConsume() (r result.Error) { + defer result.Recovery(&r) for streamName, consumers := range c.consumers { for consumerName, consumer := range consumers { assert.If(c.jobs[streamName] == nil, "stream not found, stream=%s", streamName) @@ -440,7 +191,7 @@ func (c *Client) doConsume() (r error) { concurrent = lo.FromPtr(consumer.Config.Concurrent) } if concurrent < DefaultMinConcurrent || concurrent > DefaultMaxConcurrent { - return errors.Errorf("concurrent must be in the range of %d-%d", DefaultMinConcurrent, DefaultMaxConcurrent) + return r.WithErrorf("concurrent must be in the range of %d-%d", DefaultMinConcurrent, DefaultMaxConcurrent) } logger.Info().Func(func(e *zerolog.Event) { @@ -457,13 +208,13 @@ func (c *Client) doConsume() (r error) { c.p.Lc.BeforeStop(lifecycle.WrapNoCtxErr(con.Stop)) } } - return r + return } func (c *Client) Start() error { - assert.Exit(c.initStream()) - assert.Exit(c.initConsumer()) - assert.Exit(c.doConsume()) + assert.Exit(c.initStream().Err()) + assert.Exit(c.initConsumer().Err()) + assert.Exit(c.doConsume().Err()) return nil } diff --git a/component/cloudevent/config.go b/component/cloudevent/config.go index b1a6aaff..59d717c0 100644 --- a/component/cloudevent/config.go +++ b/component/cloudevent/config.go @@ -1,90 +1,61 @@ package cloudevent import ( - "context" "fmt" "time" - "google.golang.org/protobuf/proto" - yaml "gopkg.in/yaml.v3" - "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/buildinfo/version" "github.com/pubgo/funk/v2/errors" "github.com/pubgo/funk/v2/typex" + yaml "gopkg.in/yaml.v3" ) const ( - DefaultPrefix = "acj" - DefaultTimeout = 15 * time.Second - DefaultMaxRetry = 3 - DefaultRetryBackoff = time.Second - DefaultSenderKey = "sender" - DefaultCloudEventDelayKey = "__cloudevent_delay_run_at" - DefaultJobName = "default" - DefaultConcurrent = 100 - DefaultMaxConcurrent = 1000 - DefaultMinConcurrent = 1 + DefaultPrefix = "acj" + DefaultTimeout = 15 * time.Second + DefaultMaxRetry = 3 + DefaultRetryBackoff = time.Second + SenderHeaderKey = "__cloudevent_sender" + DelayHeaderKey = "__cloudevent_delay_run_at" + DefaultJobName = "default" + DefaultConcurrent = 100 + DefaultMaxConcurrent = 1000 + DefaultMinConcurrent = 1 ) var senderValue = fmt.Sprintf("%s/%s", version.Project(), version.Version()) type Config struct { - // Streams: nats stream config - Streams map[string]*StreamConfig `yaml:"streams"` - - // Consumers: nats consumer config + Streams map[string]*StreamConfig `yaml:"streams"` Consumers map[string]typex.YamlListType[*ConsumerConfig] `yaml:"consumers"` } type StreamConfig struct { - // Storage jetstream.StorageType - Storage string `yaml:"storage"` - - // Subjects stream subscribe subject, e.g. nvr.speaker.* without prefix + Storage string `yaml:"storage"` Subjects typex.YamlListType[string] `yaml:"subjects"` } type ConsumerConfig struct { - // Consumer name without prefix - Consumer *string `yaml:"consumer"` - - // Concurrent default: 100 - Concurrent *int `yaml:"concurrent"` - - // Stream name without prefix - Stream string `yaml:"stream"` - - // Subjects config - Subjects typex.YamlListType[*strOrJobConfig] `yaml:"subjects"` - - // Job event config - Job *JobEventConfig `yaml:"job"` + Consumer *string `yaml:"consumer"` + Concurrent *int `yaml:"concurrent"` + Stream string `yaml:"stream"` + Subjects typex.YamlListType[*strOrJobConfig] `yaml:"subjects"` + Job *JobEventConfig `yaml:"job"` } type JobEventConfig struct { - // Name subject name - Name *string `yaml:"name"` - - // Timeout job executor timeout, default: DefaultTimeout - Timeout *time.Duration `yaml:"timeout"` - - // MaxRetry max retries, default: DefaultMaxRetry - MaxRetry *int `yaml:"max_retries"` - - // RetryBackoff retry backoff, default: DefaultRetryBackoff + Name *string `yaml:"name"` + Timeout *time.Duration `yaml:"timeout"` + MaxRetry *int `yaml:"max_retries"` RetryBackoff *time.Duration `yaml:"retry_backoff"` } type jobEventHandler struct { - // job name - name string - - // job handler - handler func(ctx context.Context, args proto.Message) error - - // job config - cfg *JobEventConfig + name string + manager *handlerManager + cfg *JobEventConfig + interceptors []SubInterceptor } type strOrJobConfig JobEventConfig diff --git a/component/cloudevent/context.go b/component/cloudevent/context.go index ee147485..232abdb7 100644 --- a/component/cloudevent/context.go +++ b/component/cloudevent/context.go @@ -5,49 +5,35 @@ import ( "net/http" "time" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + "github.com/rs/xid" "github.com/samber/lo" "google.golang.org/protobuf/proto" - - "github.com/pubgo/funk/v2" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" ) type Context struct { - // Header jetstream.Headers(). - Header http.Header - - // NumDelivered jetstream.MsgMetadata{}.NumDelivered + Header http.Header NumDelivered uint64 - - // NumPending jetstream.MsgMetadata{}.NumPending - NumPending uint64 - - // Timestamp jetstream.MsgMetadata{}.Timestamp - Timestamp time.Time - - // Stream jetstream.MsgMetadata{}.Stream - Stream string - - // Consumer jetstream.MsgMetadata{}.Consumer - Consumer string - - // Subject|Topic name jetstream.Msg().Subject() - Subject string - - // Config job config from config file or default - Config *JobEventConfig + NumPending uint64 + Timestamp time.Time + Stream string + Consumer string + Subject string + Config *JobEventConfig } -var cloudeventCtxKey = lo.ToPtr(funk.Void{}) +type ctxKey int + +const cloudeventCtxKey ctxKey = 1 -func createCtxWithContext(parent context.Context, ctx *Context) context.Context { +func createCtxWithSubjectContext(parent context.Context, ctx *Context) context.Context { if parent == nil { parent = context.Background() } return context.WithValue(parent, cloudeventCtxKey, ctx) } -func GetEventContext(ctx context.Context) *Context { +func GetContext(ctx context.Context) *Context { if ctx == nil { return nil } @@ -60,24 +46,7 @@ func GetEventContext(ctx context.Context) *Context { return evtCtx } -var pushEventCtxKey = lo.ToPtr(struct{}{}) - -func withOptions(ctx context.Context, opts ...*cloudeventpb.PushEventOptions) context.Context { - if len(opts) == 0 { - return ctx - } - - oldOpts, ok := ctx.Value(pushEventCtxKey).(*cloudeventpb.PushEventOptions) - if !ok { - oldOpts = new(cloudeventpb.PushEventOptions) - } - - for i := range opts { - proto.Merge(oldOpts, opts[i]) - } - - return context.WithValue(ctx, pushEventCtxKey, oldOpts) -} +var pushEventCtxKey = xid.New().String() func WithPushOpt(opts ...func(opt *cloudeventpb.PushEventOptions)) *cloudeventpb.PushEventOptions { var opt cloudeventpb.PushEventOptions @@ -87,20 +56,64 @@ func WithPushOpt(opts ...func(opt *cloudeventpb.PushEventOptions)) *cloudeventpb return &opt } -func getOptions(ctx context.Context, opts ...*cloudeventpb.PushEventOptions) *cloudeventpb.PushEventOptions { - evtOpt := new(cloudeventpb.PushEventOptions) - opt, ok := ctx.Value(pushEventCtxKey).(*cloudeventpb.PushEventOptions) - if ok { +func getOptions(ctx context.Context, opts ...PubOpt) *PubOptions { + evtOpt := new(PubOptions) + if opt, ok := ctx.Value(pushEventCtxKey).(*PubOptions); ok { evtOpt = opt } for _, o := range opts { - proto.Merge(evtOpt, o) + if o == nil { + continue + } + o(evtOpt) } if evtOpt.GetMsgId() == "" { evtOpt.MsgId = nil } + if evtOpt.ContentType == nil { + evtOpt.ContentType = lo.ToPtr("application/json") + } + + if evtOpt.Sender == nil { + evtOpt.Sender = lo.ToPtr(senderValue) + } + return evtOpt } + +func ProtoPubOpts(opts ...*cloudeventpb.PushEventOptions) []PubOpt { + if len(opts) == 0 { + return nil + } + return []PubOpt{func(po *PubOptions) { + for _, o := range opts { + if o == nil { + continue + } + proto.Merge(po, o) + } + }} +} + +func ProtoRegisterOpts(opts ...*cloudeventpb.RegisterJobOptions) RegisterOpt { + return func(ro *RegisterJobOptions) { + if ro.Opts == nil { + ro.Opts = new(cloudeventpb.RegisterJobOptions) + } + for _, o := range opts { + if o == nil { + continue + } + proto.Merge(ro.Opts, o) + } + } +} + +func WithSubInterceptors(interceptors ...SubInterceptor) RegisterOpt { + return func(ro *RegisterJobOptions) { + ro.Interceptors = append(ro.Interceptors, interceptors...) + } +} diff --git a/component/cloudevent/errors.go b/component/cloudevent/errors.go index c625a391..2cbd64fd 100644 --- a/component/cloudevent/errors.go +++ b/component/cloudevent/errors.go @@ -10,6 +10,7 @@ import ( var ( errReject = errors.New("cloudevent: reject retry and discard msg") errRedeliveryStr = "cloudevent: redelivery message with custom delay duration" + errForceRetry = errors.New("cloudevent: force redelivery message with no delay") ) func Reject(errs ...error) error { @@ -21,11 +22,7 @@ func Reject(errs ...error) error { } func isRejectErr(err error) bool { - if err == nil { - return false - } - - return errors.Is(err, errReject) + return err != nil && errors.Is(err, errReject) } type errRedelivery struct { @@ -56,4 +53,20 @@ func isRedeliveryErr(err error) *errRedelivery { return nil } -// TODO force retry +type forceRetryError struct{} + +func (err forceRetryError) Error() string { + return errForceRetry.Error() +} + +func ForceRetry(errs ...error) error { + reason := "force_retry" + if len(errs) > 0 { + reason = errs[0].Error() + } + return errors.Wrap(&forceRetryError{}, reason) +} + +func isForceRetry(err error) bool { + return err != nil && errors.As(err, new(forceRetryError)) +} diff --git a/component/cloudevent/publisher.go b/component/cloudevent/publisher.go index 65d91541..f978c0ed 100644 --- a/component/cloudevent/publisher.go +++ b/component/cloudevent/publisher.go @@ -6,156 +6,58 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/pubgo/funk/v2/ctxutil" + "github.com/pubgo/funk/v2/errors" + "github.com/pubgo/funk/v2/result" + "github.com/pubgo/funk/v2/typex" "github.com/rs/xid" "github.com/rs/zerolog" "github.com/samber/lo" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - - "github.com/pubgo/funk/v2/ctxutil" - "github.com/pubgo/funk/v2/errors" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" - "github.com/pubgo/funk/v2/result" - "github.com/pubgo/funk/v2/stack" - "github.com/pubgo/funk/v2/try" - "github.com/pubgo/funk/v2/typex" ) -func PushEvent[T any](handler func(*Client, context.Context, T, ...*cloudeventpb.PushEventOptions) (*PubAckInfo, error), jobCli *Client, ctx context.Context, t T, opts ...*cloudeventpb.PushEventOptions) chan result.Result[*PubAckInfo] { - errChan := make(chan result.Result[*PubAckInfo]) - timeout := ctxutil.GetTimeout(ctx) - now := time.Now() - fnCaller := stack.Caller(1).String() - - // clone ctx and recalculate timeout - ctx = lo.T2(ctxutil.Clone(ctx, DefaultTimeout)).A - go func() { - getPubAck := func() (pubAck *PubAckInfo, err error) { - err = try.Try(func() error { - pubAck, err = handler(jobCli, ctx, t, opts...) - return err - }) - return pubAck, err - } - pubAck, err := getPubAck() - err = errors.IfErr(err, func(err error) error { - logger.Err(err, ctx).Func(func(e *zerolog.Event) { - if timeout != nil { - e.Str("timeout", timeout.String()) - } - - e.Str("fn_caller", fnCaller) - e.Any("input", t) - e.Str("stack", stack.CallerWithFunc(handler).String()) - e.Str("cost", time.Since(now).String()) - e.Msg("failed to push event msg to nats stream") - }) - return err - }) - errChan <- result.Wrap(pubAck, err) - }() - return errChan +func Publish(jobCli *Client, ctx context.Context, topic string, args proto.Message, interceptors []PubInterceptor, opts ...PubOpt) result.Result[*PubAckInfo] { + return jobCli.Publish(ctx, topic, args, interceptors, opts...) } -// PushRpcEvent push event async -func PushRpcEvent[T proto.Message](handler RpcEventHandler[T], ctx context.Context, t T, opts ...*cloudeventpb.PushEventOptions) chan error { - // clone ctx and recalculate timeout - ctx = lo.T2(ctxutil.Clone(ctx, DefaultTimeout)).A - ctx = withOptions(ctx, opts...) - - fnCaller := stack.Caller(1).String() - errChan := make(chan error) - timeout := ctxutil.GetTimeout(ctx) - now := time.Now() - - pushEventBasic := func(handler RpcEventHandler[T], ctx context.Context) error { - err := try.Try(func() error { return lo.T2(handler(ctx, t)).B }) - if err == nil { - return nil - } - - logger.Err(err, ctx).Func(func(e *zerolog.Event) { - if timeout != nil { - e.Str("timeout", timeout.String()) - } - - e.Str("fn_caller", fnCaller) - e.Any("input", t) - e.Str("stack", stack.CallerWithFunc(handler).String()) - e.Str("cost", time.Since(now).String()) - e.Msg("failed to push event msg to nats stream") - }) - return err - } - - go func() { errChan <- pushEventBasic(handler, ctx) }() - return errChan +func (c *Client) Publish(ctx context.Context, topic string, args proto.Message, interceptors []PubInterceptor, opts ...PubOpt) result.Result[*PubAckInfo] { + return c.publish(ctx, topic, args, interceptors, opts...) } -func (c *Client) Publish(ctx context.Context, topic string, args proto.Message, opts ...*cloudeventpb.PushEventOptions) (*PubAckInfo, error) { - return c.publish(ctx, topic, args, opts...) -} +func (c *Client) doPublish(ctx context.Context, topic string, args proto.Message, opts *PubOptions) (r result.Result[*PubAckInfo]) { + defer result.Recovery(&r) -func (c *Client) publish(ctx context.Context, topic string, args proto.Message, opts ...*cloudeventpb.PushEventOptions) (_ *PubAckInfo, gErr error) { - defer result.RecoveryErr(&gErr) - timeout := ctxutil.GetTimeout(ctx) - now := time.Now() msgId := xid.New().String() - var pushEventOpt *cloudeventpb.PushEventOptions - var pubActInfo *jetstream.PubAck - - defer func() { - msgFn := func(e *zerolog.Event) { - e.Str("pub_topic", topic) - e.Str("pub_start", now.String()) - e.Any("pub_args", args) - e.Str("pub_cost", time.Since(now).String()) - e.Str("pub_msg_id", msgId) - e.Any("pub_ack_info", pubActInfo) - if timeout != nil { - e.Str("timeout", timeout.String()) - } - } - if gErr == nil { - logger.Info(ctx).Func(msgFn).Msg("succeed to publish cloud event job to stream") - } else { - logger.Err(gErr, ctx).Func(msgFn).Msg("failed to publish cloud event job to stream") - } - }() - - pushEventOpt = getOptions(ctx, opts...) - if pushEventOpt.MsgId != nil { - msgId = pushEventOpt.GetMsgId() + if opts.MsgId != nil { + msgId = opts.GetMsgId() } - proxy := result.ErrProxyOf(&gErr) pb := result.Wrap(anypb.New(args)). - Log(func(e result.Event) { - e.Msg("failed to marshal args to any proto") + MapErr(func(err error) error { + return errors.Wrap(err, "failed to marshal args to any proto") }). - UnwrapOrThrow(&proxy) - if proxy.IsErr() { + UnwrapOrThrow(&r) + if r.IsErr() { return } - // TODO get parent event info from ctx data := result.Wrap(proto.Marshal(pb)). - Log(func(e result.Event) { - e.Msg("failed to marshal any proto to bytes") + MapErr(func(err error) error { + return errors.Wrap(err, "failed to marshal any proto to bytes") }). - UnwrapOrThrow(&proxy) - if proxy.IsErr() { + UnwrapOrThrow(&r) + if r.IsErr() { return } - // subject|topic name topic = c.subjectName(topic) header := typex.DoBlock1(func() nats.Header { header := nats.Header{ - DefaultSenderKey: []string{senderValue}, - DefaultCloudEventDelayKey: []string{encodeDelayTime(pushEventOpt.DelayDur.AsDuration())}, + SenderHeaderKey: []string{lo.FromPtr(opts.Sender)}, + DelayHeaderKey: []string{encodeDelayTime(opts.Delay)}, } - for k, v := range pushEventOpt.Metadata { + for k, v := range opts.Metadata { header.Add(k, v) } return header @@ -163,18 +65,51 @@ func (c *Client) publish(ctx context.Context, topic string, args proto.Message, msg := &nats.Msg{Subject: topic, Data: data, Header: header} jetOpts := append([]jetstream.PublishOpt{}, jetstream.WithMsgID(msgId)) - pubActInfo = result.Wrap(c.js.PublishMsg(ctx, msg, jetOpts...)). - Log(func(e result.Event) { - e.Msgf("failed to publish msg to stream, topic=%s msg_id=%s", topic, msgId) + pubActInfo := result.Wrap(c.js.PublishMsg(ctx, msg, jetOpts...)). + MapErr(func(err error) error { + return errors.Wrapf(err, "failed to publish msg to jetstream, topic=%s msg_id=%s", topic, msgId) }). - UnwrapOrThrow(&proxy) - if gErr != nil { + UnwrapOrThrow(&r) + if r.IsErr() { return } - return &PubAckInfo{ + return r.WithValue(&PubAckInfo{ AckInfo: pubActInfo, Header: header, MsgId: msgId, - }, nil + }) +} + +func (c *Client) publish(ctx context.Context, topic string, args proto.Message, interceptors []PubInterceptor, opts ...PubOpt) (r result.Result[*PubAckInfo]) { + timeout := ctxutil.GetTimeout(ctx) + now := time.Now() + logFn := func(e *zerolog.Event) { + e.Str("topic", topic) + e.Str("start_at", now.String()) + e.Any("args", args) + e.Str("cost", time.Since(now).String()) + e.Any("ack_info", r.UnwrapOrEmpty()) + if timeout != nil { + e.Str("timeout", timeout.String()) + } + } + + defer func() { + if r.IsOK() { + logger.Info(ctx).Func(logFn).Msg("succeed to publish cloudevent msg to jetstream") + } else { + logger.Err(r.Err(), ctx).Func(logFn).Msg("failed to publish cloudevent msg to jetstream") + } + }() + + interceptor := func(ctx context.Context, topic string, args proto.Message, opts *PubOptions, handler func(ctx context.Context, topic string, args proto.Message, opts *PubOptions) result.Result[*PubAckInfo]) result.Result[*PubAckInfo] { + return handler(ctx, topic, args, opts) + } + for i := len(interceptors) - 1; i >= 0; i-- { + interceptor = interceptors[i](interceptor) + } + + pushEventOpt := getOptions(ctx, opts...) + return interceptor(ctx, topic, args, pushEventOpt, c.doPublish) } diff --git a/component/cloudevent/register.go b/component/cloudevent/register.go index 3490d1f6..dc910cb3 100644 --- a/component/cloudevent/register.go +++ b/component/cloudevent/register.go @@ -2,8 +2,6 @@ package cloudevent import ( "context" - "fmt" - "reflect" "github.com/rs/zerolog" "github.com/samber/lo" @@ -25,54 +23,54 @@ func WrapHandler[Req, Rsp proto.Message](handler func(ctx context.Context, req R func init() { vars.Register("cloudevent.default_config", func() any { return map[string]any{ - "default_prefix": DefaultPrefix, - "default_timeout": DefaultTimeout, - "default_max_retry": DefaultMaxRetry, - "default_retry_backoff": DefaultRetryBackoff, - "default_job_name": DefaultJobName, - "DefaultCloudEventDelayKey": DefaultCloudEventDelayKey, + "default_prefix": DefaultPrefix, + "default_timeout": DefaultTimeout, + "default_max_retry": DefaultMaxRetry, + "default_retry_backoff": DefaultRetryBackoff, + "default_job_name": DefaultJobName, + "DelayHeaderKey": DelayHeaderKey, + "SenderHeaderKey": SenderHeaderKey, } }) } -func RegisterJobHandler[T proto.Message](jobCli *Client, jobName, topic string, handler EventHandler[T], opts ...*cloudeventpb.RegisterJobOptions) { - assert.Fn(reflect.TypeOf(jobCli.subjects[topic]) != reflect.TypeOf(lo.Empty[T]()), func() error { - return fmt.Errorf("type not match, topic-type=%s handler-input-type=%s", reflect.TypeOf(jobCli.subjects[topic]).String(), reflect.TypeOf(lo.Empty[T]()).String()) - }) - +func RegisterJobHandler[T proto.Message](jobCli *Client, jobName, topic string, handler Handler[T], opts ...RegisterOpt) { if jobName == "" { jobName = DefaultJobName } - jobCli.registerJobHandler(jobName, topic, func(ctx context.Context, args proto.Message) error { return handler(ctx, args.(T)) }, opts...) + eventHandler := func(ctx context.Context, args proto.Message) error { return handler(ctx, args.(T)) } + jobCli.registerJobHandler(jobName, topic, eventHandler, opts...) } -func (c *Client) registerJobHandler(jobName, topic string, handler EventHandler[proto.Message], opts ...*cloudeventpb.RegisterJobOptions) { - assert.If(handler == nil, "job handler is nil") +func (c *Client) registerJobHandler(jobName, topic string, handler Handler[proto.Message], opts ...RegisterOpt) { + assert.If(handler == nil, "job manager is nil") assert.If(c.subjects[topic] == nil, "topic:%s not found", topic) - evtOpt := new(cloudeventpb.RegisterJobOptions) + jobOpt := &RegisterJobOptions{Opts: new(cloudeventpb.RegisterJobOptions)} for _, o := range opts { - proto.Merge(evtOpt, o) + o(jobOpt) } - if lo.FromPtr(evtOpt.JobName) != "" { - jobName = lo.FromPtr(evtOpt.JobName) + if name := lo.FromPtr(jobOpt.Opts.JobName); name != "" { + jobName = name } - if c.handlers[jobName] == nil { - c.handlers[jobName] = map[string]EventHandler[proto.Message]{} + if c.jobManagers[jobName] == nil { + c.jobManagers[jobName] = &jobManager{managers: make(map[string]*handlerManager), interceptors: jobOpt.Interceptors} + } else if len(jobOpt.Interceptors) > 0 { + c.jobManagers[jobName].interceptors = append(c.jobManagers[jobName].interceptors, jobOpt.Interceptors...) } topic = c.subjectName(topic) - assert.If(c.handlers[jobName][topic] != nil, "job handler already registered, job_name=%s, topic=%s", jobName, topic) + assert.If(c.jobManagers[jobName].managers[topic] != nil, "job manager already registered, job_name=%s, topic=%s", jobName, topic) - c.handlers[jobName][topic] = handler + c.jobManagers[jobName].managers[topic] = &handlerManager{handler: handler} logger.Info().Func(func(e *zerolog.Event) { - e.Str("job_name", jobName) + e.Str("job", jobName) e.Str("topic", topic) - e.Str("job_handler", stack.CallerWithFunc(handler).String()) - e.Msg("register cloud job handler") + e.Str("handler", stack.CallerWithFunc(handler).String()) + e.Msg("register cloudevent handler") }) } diff --git a/component/cloudevent/subscriber.go b/component/cloudevent/subscriber.go new file mode 100644 index 00000000..c0301970 --- /dev/null +++ b/component/cloudevent/subscriber.go @@ -0,0 +1,252 @@ +package cloudevent + +import ( + "context" + "net/http" + "time" + + "github.com/nats-io/nats.go/jetstream" + "github.com/panjf2000/ants/v2" + "github.com/pubgo/funk/v2/assert" + "github.com/pubgo/funk/v2/errors" + "github.com/pubgo/funk/v2/log" + "github.com/pubgo/funk/v2/result" + "github.com/pubgo/funk/v2/stack" + "github.com/pubgo/funk/v2/try" + "github.com/rs/zerolog" + "github.com/samber/lo" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func (c *Client) doConsumeHandler(streamName, consumerName string, jobSubjects map[string]*jobEventHandler, concurrent int) func(msg jetstream.Msg) { + handler := func(msg jetstream.Msg) { + now := time.Now() + addMsgInfo := func(e *zerolog.Event) { + e.Str("stream", streamName) + e.Str("consumer", consumerName) + e.Any("header", msg.Headers()) + e.Any("msg_id", msg.Headers().Get(jetstream.MsgIDHeader)) + e.Str("subject", msg.Subject()) + e.Str("msg_received_time", now.String()) + e.Str("job_cost", time.Since(now).String()) + } + + logger.Debug().Func(addMsgInfo).Msg("received cloud job manager") + + handlerDelayJob := func() (r result.Result[bool]) { + dur := decodeDelayTime(msg.Headers().Get(DelayHeaderKey)). + MapErr(func(err error) error { + return errors.Wrap(err, "failed to parse job delay time") + }). + UnwrapOrThrow(&r) + if r.IsErr() { + return + } + + if dur <= 0 { + return r.WithValue(false) + } + + if result.Throw(&r, msg.NakWithDelay(dur)) { + return + } + + return r.WithValue(true) + } + + delayRet := handlerDelayJob(). + IfErr(func(err error) { + logger.Err(err).Func(addMsgInfo).Msg("failed to handle cloud delay job and no ack") + }). + IfOK(func(b bool) { + if b { + logger.Info().Func(addMsgInfo).Msg("redeliver the message after the given delay") + } + }) + + if delayRet.IsErr() || delayRet.Unwrap() { + return + } + + job := jobSubjects[msg.Subject()] + if job == nil { + logger.Error().Func(addMsgInfo).Msg("failed to find subject job manager") + return + } + + meta := result.Wrap(msg.Metadata()). + IfErr(func(err error) { + logger.Err(err).Func(addMsgInfo).Msg("failed to parse nats stream msg metadata") + }) + if meta.IsErr() { + return + } + + cfg := job.cfg + checkErrAndLog := func(err error, msg string) { + if err == nil { + return + } + + logger.Err(err). + Str("fn_caller", stack.Caller(1).String()). + Func(addMsgInfo). + Any("metadata", meta). + Any("config", cfg). + Any("msg_received_time", now.String()). + Str("job_cost", time.Since(now).String()). + Msg(msg) + } + + err := try.Try(func() error { return c.doHandler(meta.Unwrap(), msg, job, cfg).Err() }) + if err == nil { + checkErrAndLog(msg.Ack(), "failed to do msg ack with manager ok") + return + } + + if isRejectErr(err) { + checkErrAndLog(msg.TermWithReason("reject by caller"), "failed to do msg ack with reject err") + return + } + + if isForceRetry(err) { + checkErrAndLog(msg.Nak(), "force_retry: failed to reply nak msg") + return + } + + backoff := lo.FromPtr(cfg.RetryBackoff) + maxRetries := lo.FromPtr(cfg.MaxRetry) + + if err1 := isRedeliveryErr(err); err1 != nil { + backoff = err1.delay + } + + if meta.Unwrap().NumDelivered < uint64(maxRetries) { + logger.Warn(). + Err(err). + Func(addMsgInfo). + Any("metadata", meta). + Msg("retry nats stream cloud job manager") + checkErrAndLog(msg.NakWithDelay(backoff), "failed to retry msg with delay nak") + return + } + + checkErrAndLog(err, "failed to do manager cloud job") + checkErrAndLog(msg.Ack(), "failed to do msg ack with manager error") + } + + pool := assert.Must1(ants.NewPool( + concurrent, + ants.WithLogger(log.NewStd(logger)), + ants.WithNonblocking(false), + )) + return func(msg jetstream.Msg) { + if pool.Running() == concurrent { + logger.Warn().Func(func(e *zerolog.Event) { + e.Int("concurrent", concurrent) + e.Str("stream", streamName) + e.Str("consumer", consumerName) + e.Msg("concurrent limit occurred, please check the concurrent limit") + }) + } + if err := pool.Submit(func() { handler(msg) }); err != nil { + logger.Err(err).Func(func(e *zerolog.Event) { + e.Str("stream", streamName) + e.Str("consumer", consumerName) + e.Msg("failed to submit job to pool") + }) + } + } +} + +func (c *Client) doErrHandler(streamName, consumerName string) jetstream.PullConsumeOpt { + return jetstream.ConsumeErrHandler(func(consumeCtx jetstream.ConsumeContext, err error) { + logger.Err(err). + Str("stream", streamName). + Str("consumer", consumerName). + Msg("nats consumer error") + }) +} + +func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { + timeout := lo.FromPtr(cfg.Timeout) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + ctx = log.UpdateFieldsCtx(ctx, log.Fields{ + "sub_subject": msg.Subject(), + "sub_stream": meta.Stream, + "sub_consumer": meta.Consumer, + "sub_msg_id": msg.Headers().Get(jetstream.MsgIDHeader), + SenderHeaderKey: msg.Headers().Get(SenderHeaderKey), + }) + + msgCtx := &Context{ + Header: http.Header(msg.Headers()), + NumDelivered: meta.NumDelivered, + NumPending: meta.NumPending, + Timestamp: meta.Timestamp, + Stream: meta.Stream, + Consumer: meta.Consumer, + Subject: msg.Subject(), + Config: cfg, + } + + now := time.Now() + var args any + defer gErr.InspectErr(func(err error) { + logger.Err(err).Func(func(e *zerolog.Event) { + e.Any("context", msgCtx) + e.Any("args", args) + e.Str("timeout", timeout.String()) + e.Str("start_time", now.String()) + e.Str("job_cost", time.Since(now).String()) + e.Msg("failed to do cloud job manager") + }) + }) + + var pb anypb.Any + if result.ErrOf(proto.Unmarshal(msg.Data(), &pb)). + MapErr(func(err error) error { + return errors.WrapTags(err, errors.Tags{ + "msg": "failed to unmarshal stream msg data to any proto", + "args": string(msg.Data()), + }) + }). + Throw(&gErr) { + return + } + args = &pb + + dst := result.Wrap(anypb.UnmarshalNew(args.(*anypb.Any), proto.UnmarshalOptions{})). + MapErr(func(err error) error { + return errors.WrapTags(err, errors.Tags{ + "msg": "failed to unmarshal any proto to proto msg", + "args": args, + }) + }). + UnwrapOrThrow(&gErr) + if gErr.IsErr() { + return + } + + ctx = createCtxWithSubjectContext(ctx, msgCtx) + + interceptor := func(ctx context.Context, args proto.Message, handler func(ctx context.Context, args proto.Message) error) error { + return handler(ctx, args) + } + + for i := len(job.interceptors) - 1; i >= 0; i-- { + interceptor = job.interceptors[i](interceptor) + } + + return result.ErrOf(interceptor(ctx, dst, job.manager.handler)). + MapErr(func(err error) error { + return errors.WrapTags(err, errors.Tags{ + "msg": "failed to do cloud job manager", + "args": args, + "any_pb": dst, + }) + }) +} diff --git a/component/cloudevent/util.go b/component/cloudevent/util.go index 81cf2d80..d8cef89b 100644 --- a/component/cloudevent/util.go +++ b/component/cloudevent/util.go @@ -7,15 +7,16 @@ import ( "time" "github.com/nats-io/nats.go/jetstream" - "github.com/rs/zerolog" - "github.com/samber/lo" - "google.golang.org/protobuf/reflect/protoreflect" - "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/errors" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" "github.com/pubgo/funk/v2/protoutils" "github.com/pubgo/funk/v2/result" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + cloudeventoptionpb "github.com/pubgo/funk/v2/proto/cloudeventoption" + "github.com/rs/zerolog" + "github.com/samber/lo" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/durationpb" ) func getStorageType(name string) jetstream.StorageType { @@ -82,24 +83,33 @@ func handleSubjectName(name, prefix string) string { return fmt.Sprintf("%s%s", prefix, name) } -func encodeDelayTime(duration time.Duration) string { - return strconv.Itoa(int(time.Now().Add(duration).UnixMilli())) +func encodeDelayTime(duration *durationpb.Duration) string { + if duration == nil { + return "" + } + + return strconv.Itoa(int(time.Now().Add(duration.AsDuration()).UnixMilli())) } func decodeDelayTime(delayTime string) (r result.Result[time.Duration]) { + delayTime = strings.TrimSpace(delayTime) + if delayTime == "" { + return r.WithValue(0) + } + tt := result.Wrap(strconv.Atoi(delayTime)). MapErr(func(err error) error { - return errors.Wrapf(err, "failed to parse cloud event job delay time, time=%s", delayTime) + return errors.Wrapf(err, "failed to parse cloudevent job delay time, time=%s", delayTime) }) - return result.MapTo(tt, func(t int) time.Duration { - return time.Until(time.UnixMilli(int64(t))) + return result.MapTo(tt, func(value int) time.Duration { + return time.Until(time.UnixMilli(int64(value))) }) } type subjectOpt struct { - *cloudeventpb.CloudEventServiceOptions - *cloudeventpb.CloudEventMethodOptions + Job *cloudeventpb.CloudEventServiceOptions + Subject *cloudeventpb.CloudEventMethodOptions } func registerSubject(subjects map[string]*cloudeventpb.CloudEventMethodOptions, subject, operation string, data *cloudeventpb.CloudEventMethodOptions) any { @@ -124,7 +134,7 @@ func registerSubject(subjects map[string]*cloudeventpb.CloudEventMethodOptions, func getAllSubject() map[string]*cloudeventpb.CloudEventMethodOptions { subjects := make(map[string]*cloudeventpb.CloudEventMethodOptions) for _, opt := range getAllSubjectOptions() { - registerSubject(subjects, opt.CloudEventServiceOptions.Name, *opt.Operation, opt.CloudEventMethodOptions) + registerSubject(subjects, opt.Subject.Name, *opt.Subject.Operation, opt.Subject) } return subjects } @@ -132,27 +142,27 @@ func getAllSubject() map[string]*cloudeventpb.CloudEventMethodOptions { func getAllSubjectOptions() []subjectOpt { var opts []subjectOpt protoutils.EachService(func(desc protoreflect.FileDescriptor, srv protoreflect.ServiceDescriptor) { - if !protoutils.HasExtension(srv.Options(), cloudeventpb.E_Job) { + if !protoutils.HasExtension(srv.Options(), cloudeventoptionpb.E_Job) { return } - jobOpt := protoutils.GetExtension[cloudeventpb.CloudEventServiceOptions](srv.Options(), cloudeventpb.E_Job) + jobOpt := protoutils.GetExtension[cloudeventpb.CloudEventServiceOptions](srv.Options(), cloudeventoptionpb.E_Job) if jobOpt == nil { return } protoutils.EachServiceMethod(srv, func(mth protoreflect.MethodDescriptor) { - if !protoutils.HasExtension(mth.Options(), cloudeventpb.E_Subject) { + if !protoutils.HasExtension(mth.Options(), cloudeventoptionpb.E_Subject) { return } - subOpt := protoutils.GetExtension[cloudeventpb.CloudEventMethodOptions](mth.Options(), cloudeventpb.E_Subject) + subOpt := protoutils.GetExtension[cloudeventpb.CloudEventMethodOptions](mth.Options(), cloudeventoptionpb.E_Subject) if subOpt == nil { return } subOpt.Operation = lo.ToPtr(fmt.Sprintf("/%s/%s", srv.FullName(), mth.Name())) - opts = append(opts, subjectOpt{CloudEventServiceOptions: jobOpt, CloudEventMethodOptions: subOpt}) + opts = append(opts, subjectOpt{Job: jobOpt, Subject: subOpt}) }) }) return opts diff --git a/proto/cloudevent/options.pb.go b/proto/cloudevent/options.pb.go deleted file mode 100644 index 1474c5fb..00000000 --- a/proto/cloudevent/options.pb.go +++ /dev/null @@ -1,124 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.35.2 -// protoc v6.32.1 -// source: cloudevent/options.proto - -package cloudeventpb - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - descriptorpb "google.golang.org/protobuf/types/descriptorpb" - reflect "reflect" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -var file_cloudevent_options_proto_extTypes = []protoimpl.ExtensionInfo{ - { - ExtendedType: (*descriptorpb.ServiceOptions)(nil), - ExtensionType: (*CloudEventServiceOptions)(nil), - Field: 10010, - Name: "lava.cloudevent.job", - Tag: "bytes,10010,opt,name=job", - Filename: "cloudevent/options.proto", - }, - { - ExtendedType: (*descriptorpb.MethodOptions)(nil), - ExtensionType: (*CloudEventMethodOptions)(nil), - Field: 10011, - Name: "lava.cloudevent.subject", - Tag: "bytes,10011,opt,name=subject", - Filename: "cloudevent/options.proto", - }, -} - -// Extension fields to descriptorpb.ServiceOptions. -var ( - // optional lava.cloudevent.CloudEventServiceOptions job = 10010; - E_Job = &file_cloudevent_options_proto_extTypes[0] -) - -// Extension fields to descriptorpb.MethodOptions. -var ( - // optional lava.cloudevent.CloudEventMethodOptions subject = 10011; - E_Subject = &file_cloudevent_options_proto_extTypes[1] -) - -var File_cloudevent_options_proto protoreflect.FileDescriptor - -var file_cloudevent_options_proto_rawDesc = []byte{ - 0x0a, 0x18, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x6c, 0x61, 0x76, 0x61, - 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x1a, 0x20, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3a, 0x5d, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x12, 0x1f, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x9a, 0x4e, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6c, 0x61, 0x76, 0x61, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x03, 0x6a, 0x6f, 0x62, 0x3a, 0x63, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, - 0x1e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x9b, 0x4e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6c, 0x61, 0x76, 0x61, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x75, 0x62, 0x67, 0x6f, 0x2f, 0x66, 0x75, - 0x6e, 0x6b, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var file_cloudevent_options_proto_goTypes = []any{ - (*descriptorpb.ServiceOptions)(nil), // 0: google.protobuf.ServiceOptions - (*descriptorpb.MethodOptions)(nil), // 1: google.protobuf.MethodOptions - (*CloudEventServiceOptions)(nil), // 2: lava.cloudevent.CloudEventServiceOptions - (*CloudEventMethodOptions)(nil), // 3: lava.cloudevent.CloudEventMethodOptions -} -var file_cloudevent_options_proto_depIdxs = []int32{ - 0, // 0: lava.cloudevent.job:extendee -> google.protobuf.ServiceOptions - 1, // 1: lava.cloudevent.subject:extendee -> google.protobuf.MethodOptions - 2, // 2: lava.cloudevent.job:type_name -> lava.cloudevent.CloudEventServiceOptions - 3, // 3: lava.cloudevent.subject:type_name -> lava.cloudevent.CloudEventMethodOptions - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 2, // [2:4] is the sub-list for extension type_name - 0, // [0:2] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_cloudevent_options_proto_init() } -func file_cloudevent_options_proto_init() { - if File_cloudevent_options_proto != nil { - return - } - file_cloudevent_types_proto_init() - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cloudevent_options_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 2, - NumServices: 0, - }, - GoTypes: file_cloudevent_options_proto_goTypes, - DependencyIndexes: file_cloudevent_options_proto_depIdxs, - ExtensionInfos: file_cloudevent_options_proto_extTypes, - }.Build() - File_cloudevent_options_proto = out.File - file_cloudevent_options_proto_rawDesc = nil - file_cloudevent_options_proto_goTypes = nil - file_cloudevent_options_proto_depIdxs = nil -} diff --git a/proto/cloudevent/types.pb.go b/proto/cloudevent/types.pb.go index 70ae0c07..85df731f 100644 --- a/proto/cloudevent/types.pb.go +++ b/proto/cloudevent/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.35.2 -// protoc v6.32.1 +// protoc v6.33.4 // source: cloudevent/types.proto package cloudeventpb @@ -72,14 +72,20 @@ type PushEventOptions struct { unknownFields protoimpl.UnknownFields // The content type for the data (optional). + // If not set, the default content type is "application/json". ContentType *string `protobuf:"bytes,1,opt,name=content_type,json=contentType,proto3,oneof" json:"content_type,omitempty"` + // The metadata passing to pub components // metadata property: // - key : the key of the message. Metadata map[string]string `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // The delay duration of the message. - DelayDur *durationpb.Duration `protobuf:"bytes,3,opt,name=delay_dur,json=delayDur,proto3,oneof" json:"delay_dur,omitempty"` + Delay *durationpb.Duration `protobuf:"bytes,3,opt,name=delay,proto3,oneof" json:"delay,omitempty"` // The message id + // If not set, the default message id is uuid. + // If set, the message id must be unique. MsgId *string `protobuf:"bytes,4,opt,name=msg_id,json=msgId,proto3,oneof" json:"msg_id,omitempty"` + // The sender of the message. + Sender *string `protobuf:"bytes,5,opt,name=sender,proto3,oneof" json:"sender,omitempty"` } func (x *PushEventOptions) Reset() { @@ -126,9 +132,9 @@ func (x *PushEventOptions) GetMetadata() map[string]string { return nil } -func (x *PushEventOptions) GetDelayDur() *durationpb.Duration { +func (x *PushEventOptions) GetDelay() *durationpb.Duration { if x != nil { - return x.DelayDur + return x.Delay } return nil } @@ -140,6 +146,13 @@ func (x *PushEventOptions) GetMsgId() string { return "" } +func (x *PushEventOptions) GetSender() string { + if x != nil && x.Sender != nil { + return *x.Sender + } + return "" +} + type CloudEventServiceOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -195,8 +208,8 @@ type CloudEventMethodOptions struct { // subject name is same with config jobs consumers // subject name sametime is topic name Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // event or api operation - Operation *string `protobuf:"bytes,4,opt,name=operation,proto3,oneof" json:"operation,omitempty"` + // cloud event operation + Operation *string `protobuf:"bytes,2,opt,name=operation,proto3,oneof" json:"operation,omitempty"` } func (x *CloudEventMethodOptions) Reset() { @@ -254,7 +267,7 @@ var file_cloudevent_types_proto_rawDesc = []byte{ 0x69, 0x73, 0x74, 0x65, 0x72, 0x4a, 0x6f, 0x62, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, - 0x0b, 0x0a, 0x09, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xc7, 0x02, 0x0a, + 0x0b, 0x0a, 0x09, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xe4, 0x02, 0x0a, 0x10, 0x50, 0x75, 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, @@ -263,32 +276,34 @@ var file_cloudevent_types_proto_rawDesc = []byte{ 0x76, 0x61, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x75, 0x73, 0x68, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3b, 0x0a, 0x09, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x5f, - 0x64, 0x75, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x01, 0x52, 0x08, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x44, 0x75, 0x72, - 0x88, 0x01, 0x01, 0x12, 0x1a, 0x0a, 0x06, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x05, 0x6d, 0x73, 0x67, 0x49, 0x64, 0x88, 0x01, 0x01, 0x1a, - 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0f, 0x0a, 0x0d, - 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x42, 0x0c, 0x0a, - 0x0a, 0x5f, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x5f, 0x64, 0x75, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x5f, - 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x22, 0x2e, 0x0a, 0x18, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x5e, 0x0a, 0x17, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x75, 0x62, 0x67, 0x6f, 0x2f, 0x66, 0x75, 0x6e, 0x6b, 0x2f, - 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x70, 0x62, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x34, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x48, 0x01, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x88, 0x01, 0x01, 0x12, 0x1a, 0x0a, 0x06, + 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x05, + 0x6d, 0x73, 0x67, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x73, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x03, 0x52, 0x06, 0x73, 0x65, 0x6e, 0x64, + 0x65, 0x72, 0x88, 0x01, 0x01, 0x1a, 0x3b, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x42, 0x09, 0x0a, + 0x07, 0x5f, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x65, 0x6e, + 0x64, 0x65, 0x72, 0x22, 0x2e, 0x0a, 0x18, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x22, 0x5e, 0x0a, 0x17, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x70, 0x75, 0x62, 0x67, 0x6f, 0x2f, 0x66, 0x75, 0x6e, 0x6b, 0x2f, 0x76, 0x32, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -314,7 +329,7 @@ var file_cloudevent_types_proto_goTypes = []any{ } var file_cloudevent_types_proto_depIdxs = []int32{ 4, // 0: lava.cloudevent.PushEventOptions.metadata:type_name -> lava.cloudevent.PushEventOptions.MetadataEntry - 5, // 1: lava.cloudevent.PushEventOptions.delay_dur:type_name -> google.protobuf.Duration + 5, // 1: lava.cloudevent.PushEventOptions.delay:type_name -> google.protobuf.Duration 2, // [2:2] is the sub-list for method output_type 2, // [2:2] is the sub-list for method input_type 2, // [2:2] is the sub-list for extension type_name diff --git a/proto/cloudevent/types.proto b/proto/cloudevent/types.proto index b47a2de0..39381c45 100644 --- a/proto/cloudevent/types.proto +++ b/proto/cloudevent/types.proto @@ -7,41 +7,42 @@ import "google/protobuf/duration.proto"; option go_package = "github.com/pubgo/funk/v2/proto/cloudevent;cloudeventpb"; message RegisterJobOptions { - optional string job_name = 1; + optional string job_name = 1; } message PushEventOptions { + // The content type for the data (optional). + // If not set, the default content type is "application/json". + optional string content_type = 1; - // The content type for the data (optional). - optional string content_type = 1; + // The metadata passing to pub components + // metadata property: + // - key : the key of the message. + map metadata = 2; - // The metadata passing to pub components + // The delay duration of the message. + optional google.protobuf.Duration delay = 3; - // metadata property: - // - key : the key of the message. - map metadata = 2; + // The message id + // If not set, the default message id is uuid. + // If set, the message id must be unique. + optional string msg_id = 4; - // The delay duration of the message. - optional google.protobuf.Duration delay_dur = 3; - - // The message id - optional string msg_id = 4; + // The sender of the message. + optional string sender = 5; } message CloudEventServiceOptions { - - // job event name is same with config jobs consumers job name - string name = 1; + // job event name is same with config jobs consumers job name + string name = 1; } // cloud event subject or topic message CloudEventMethodOptions { + // subject name is same with config jobs consumers + // subject name sametime is topic name + string name = 1; - // subject name is same with config jobs consumers - // subject name sametime is topic name - string name = 1; - - // event or api operation - optional string operation = 4; + // cloud event operation + optional string operation = 2; } - diff --git a/proto/cloudeventoption/options.pb.go b/proto/cloudeventoption/options.pb.go new file mode 100644 index 00000000..cd3c7ccb --- /dev/null +++ b/proto/cloudeventoption/options.pb.go @@ -0,0 +1,125 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.2 +// protoc v6.33.4 +// source: cloudeventoption/options.proto + +package cloudeventoptionpb + +import ( + cloudevent "github.com/pubgo/funk/v2/proto/cloudevent" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + descriptorpb "google.golang.org/protobuf/types/descriptorpb" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var file_cloudeventoption_options_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*descriptorpb.ServiceOptions)(nil), + ExtensionType: (*cloudevent.CloudEventServiceOptions)(nil), + Field: 10010, + Name: "lava.cloudevent.job", + Tag: "bytes,10010,opt,name=job", + Filename: "cloudeventoption/options.proto", + }, + { + ExtendedType: (*descriptorpb.MethodOptions)(nil), + ExtensionType: (*cloudevent.CloudEventMethodOptions)(nil), + Field: 10011, + Name: "lava.cloudevent.subject", + Tag: "bytes,10011,opt,name=subject", + Filename: "cloudeventoption/options.proto", + }, +} + +// Extension fields to descriptorpb.ServiceOptions. +var ( + // optional lava.cloudevent.CloudEventServiceOptions job = 10010; + E_Job = &file_cloudeventoption_options_proto_extTypes[0] +) + +// Extension fields to descriptorpb.MethodOptions. +var ( + // optional lava.cloudevent.CloudEventMethodOptions subject = 10011; + E_Subject = &file_cloudeventoption_options_proto_extTypes[1] +) + +var File_cloudeventoption_options_proto protoreflect.FileDescriptor + +var file_cloudeventoption_options_proto_rawDesc = []byte{ + 0x0a, 0x1e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x0f, 0x6c, 0x61, 0x76, 0x61, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x1a, 0x16, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2f, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3a, 0x5d, 0x0a, 0x03, 0x6a, + 0x6f, 0x62, 0x12, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x9a, 0x4e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6c, 0x61, 0x76, + 0x61, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6c, 0x6f, + 0x75, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x3a, 0x63, 0x0a, 0x07, 0x73, 0x75, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x9b, 0x4e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6c, + 0x61, 0x76, 0x61, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x43, + 0x6c, 0x6f, 0x75, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, + 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x75, + 0x62, 0x67, 0x6f, 0x2f, 0x66, 0x75, 0x6e, 0x6b, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x3b, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_cloudeventoption_options_proto_goTypes = []any{ + (*descriptorpb.ServiceOptions)(nil), // 0: google.protobuf.ServiceOptions + (*descriptorpb.MethodOptions)(nil), // 1: google.protobuf.MethodOptions + (*cloudevent.CloudEventServiceOptions)(nil), // 2: lava.cloudevent.CloudEventServiceOptions + (*cloudevent.CloudEventMethodOptions)(nil), // 3: lava.cloudevent.CloudEventMethodOptions +} +var file_cloudeventoption_options_proto_depIdxs = []int32{ + 0, // 0: lava.cloudevent.job:extendee -> google.protobuf.ServiceOptions + 1, // 1: lava.cloudevent.subject:extendee -> google.protobuf.MethodOptions + 2, // 2: lava.cloudevent.job:type_name -> lava.cloudevent.CloudEventServiceOptions + 3, // 3: lava.cloudevent.subject:type_name -> lava.cloudevent.CloudEventMethodOptions + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 2, // [2:4] is the sub-list for extension type_name + 0, // [0:2] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_cloudeventoption_options_proto_init() } +func file_cloudeventoption_options_proto_init() { + if File_cloudeventoption_options_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cloudeventoption_options_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 2, + NumServices: 0, + }, + GoTypes: file_cloudeventoption_options_proto_goTypes, + DependencyIndexes: file_cloudeventoption_options_proto_depIdxs, + ExtensionInfos: file_cloudeventoption_options_proto_extTypes, + }.Build() + File_cloudeventoption_options_proto = out.File + file_cloudeventoption_options_proto_rawDesc = nil + file_cloudeventoption_options_proto_goTypes = nil + file_cloudeventoption_options_proto_depIdxs = nil +} diff --git a/proto/cloudevent/options.proto b/proto/cloudeventoption/options.proto similarity index 54% rename from proto/cloudevent/options.proto rename to proto/cloudeventoption/options.proto index fba8542d..bb255e11 100644 --- a/proto/cloudevent/options.proto +++ b/proto/cloudeventoption/options.proto @@ -2,15 +2,15 @@ syntax = "proto3"; package lava.cloudevent; -import "google/protobuf/descriptor.proto"; import "cloudevent/types.proto"; +import "google/protobuf/descriptor.proto"; -option go_package = "github.com/pubgo/funk/v2/proto/cloudevent;cloudeventpb"; +option go_package = "github.com/pubgo/funk/v2/proto/cloudeventoption;cloudeventoptionpb"; extend google.protobuf.ServiceOptions { - CloudEventServiceOptions job = 10010; + CloudEventServiceOptions job = 10010; } extend google.protobuf.MethodOptions { - CloudEventMethodOptions subject = 10011; + CloudEventMethodOptions subject = 10011; } diff --git a/proto/cloudeventoption/protobuf.plugin.yaml b/proto/cloudeventoption/protobuf.plugin.yaml new file mode 100644 index 00000000..1426c6b0 --- /dev/null +++ b/proto/cloudeventoption/protobuf.plugin.yaml @@ -0,0 +1,2 @@ +plugins: + - name: go diff --git a/protobuf.yaml b/protobuf.yaml index 0945bee8..2c0a60b5 100644 --- a/protobuf.yaml +++ b/protobuf.yaml @@ -1,4 +1,4 @@ -checksum: 3cdc54c7ecf84134e660deb7ed3272063883eca9 +checksum: da0042a7fd2f896b71347db71617fc0478e20452 vendor: proto-vendor base: out: proto @@ -21,8 +21,6 @@ plugins: - name: go-enum2 linter: rules: - included_paths: [] - excluded_paths: [] enabled_rules: - core::0131::http-method - core::0131::http-body @@ -30,4 +28,3 @@ linter: disabled_rules: - all format_type: yaml - ignore_comment_disables_flag: false From 2fa239b8496dba98aa8c34982a0d2c8a458b74a3 Mon Sep 17 00:00:00 2001 From: barry Date: Wed, 17 Jun 2026 00:04:34 +0800 Subject: [PATCH 2/7] docs(cloudevent): add README and migration guide Document the dual-proto layout, codegen, runtime APIs, and breaking changes from the pre-migration API. Co-authored-by: Cursor --- AGENTS.md | 1 + component/cloudevent/README.md | 195 ++++++++++++++++++++++++++++++ component/cloudevent/README.zh.md | 192 +++++++++++++++++++++++++++++ component/cloudevent/_doc.go | 11 +- 4 files changed, 392 insertions(+), 7 deletions(-) create mode 100644 component/cloudevent/README.md create mode 100644 component/cloudevent/README.zh.md diff --git a/AGENTS.md b/AGENTS.md index 227b8e4b..328714cf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ This repository is a Go utility library module: `github.com/pubgo/funk/v2`. - [env](./env/README.md) - [stack](./stack/README.md) - [connmux](./connmux/README.md) + - [cloudevent](./component/cloudevent/README.md) ## Working rules for AI coding agents diff --git a/component/cloudevent/README.md b/component/cloudevent/README.md new file mode 100644 index 00000000..f44616ce --- /dev/null +++ b/component/cloudevent/README.md @@ -0,0 +1,195 @@ +# CloudEvent Module + +NATS JetStream-backed job runtime for protobuf-defined CloudEvents. Handlers and publishers are wired from service/method options in `.proto` files and generated by `protoc-gen-go-cloudevent2`. + +## Features + +- **JetStream jobs**: Streams, consumers, retries, and concurrency from YAML config +- **Protobuf-first**: Job name and subject/topic come from proto extensions +- **Codegen**: `Register*CloudEvent`, `*Publisher`, and subject/job key constants +- **Interceptors**: Subscribe and publish middleware chains +- **Delivery control**: `Reject`, `Redelivery`, and `ForceRetry` for consumer semantics +- **Typed results**: `Publish` returns `result.Result[*PubAckInfo]` + +## Installation + +```bash +go get github.com/pubgo/funk/v2/component/cloudevent +go install github.com/pubgo/funk/v2/cmds/protoc-gen-go-cloudevent2@latest +``` + +From this repository root: + +```bash +make protobuf # regenerates proto output +``` + +## Protobuf layout + +CloudEvent uses **two** proto packages on purpose: + +| Package | Go import | Purpose | +|---------|-----------|---------| +| `proto/cloudevent` | `github.com/pubgo/funk/v2/proto/cloudevent` (`cloudeventpb`) | Message types: `PushEventOptions`, `RegisterJobOptions`, option payloads | +| `proto/cloudeventoption` | `github.com/pubgo/funk/v2/proto/cloudeventoption` (`cloudeventoptionpb`) | Descriptor extensions `E_Job` and `E_Subject` on services/methods | + +Message types and extensions are split so application protos can import only the extension package (which re-exports the option message types) without pulling extension definitions into every generated API surface. + +### Annotate a service + +```protobuf +syntax = "proto3"; + +import "cloudeventoption/options.proto"; + +service GidInnerService { + option (lava.cloudevent.job) = { name: "gid" }; + + rpc ProxyExec(ProxyExecReq) returns (google.protobuf.Empty) { + option (lava.cloudevent.subject) = { name: "gid.proxy.exec" }; + } +} +``` + +- `job.name` must match a consumer group key in YAML config (`consumers.`). +- `subject.name` must match a subject entry under that consumer. + +### Codegen + +Add to your `protoc` invocation (see root `protobuf.yaml`): + +```text +--go-cloudevent2_out=paths=source_relative:. +``` + +Generated artifacts per annotated service include: + +- `CloudEventJobKey` — job name constant +- `CloudEventSubjectKey` — subject/topic constant +- `CloudEvent` — handler struct with `On` fields +- `RegisterCloudEvent(client, event, opts...)` — registers non-nil handlers +- `Publisher` — `{ Client, Opt, Interceptors }` with `PushEvent` methods + +## Configuration + +Jobs are declared in YAML (see `config.yaml` for shape): + +```yaml +jobs: + streams: + gid: + storage: "file" + subjects: ["gid.>"] + consumers: + gid: + - consumer: "test:gid" + stream: "gid" + subjects: "gid.proxy.exec" + job: + timeout: "1m" + max_retries: 10 +``` + +Load config and create a client with your NATS JetStream connection. Subject names in config are prefixed at runtime (`DefaultPrefix`, default `acj`). + +Defaults (overridable per job): + +| Setting | Default | +|---------|---------| +| Handler timeout | `15s` | +| Max retries | `3` | +| Retry backoff | `1s` | +| Consumer `AckWait` | `5m` | + +## Quick start + +### Register handlers (generated) + +```go +RegisterGidInnerServiceCloudEvent(jobCli, GidInnerServiceCloudEvent{ + OnProxyExec: func(ctx context.Context, req *gidpb.ProxyExecReq) error { + evt := cloudevent.GetContext(ctx) + // evt.Subject, evt.NumDelivered, evt.Config, ... + return nil + }, +}) +``` + +### Register handlers (manual) + +```go +cloudevent.RegisterJobHandler(jobCli, "gid", "gid.proxy.exec", + func(ctx context.Context, req *gidpb.ProxyExecReq) error { return nil }, + cloudevent.ProtoRegisterOpts(registerOpts...), + cloudevent.WithSubInterceptors(myInterceptor), +) +``` + +### Publish (generated publisher) + +```go +pub := GidInnerServicePublisher{ + Client: jobCli, + Opt: cloudevent.ProtoPubOpts(defaultPushOpts...), +} +ack := pub.PushProxyExecEvent(ctx, req).Unwrap() +``` + +### Publish (direct) + +```go +ack := cloudevent.Publish(jobCli, ctx, subjectKey, req, interceptors, opts...).Unwrap() +``` + +`Publish` merges `PubOpt` values, sets default `content_type` to `application/json`, and fills `sender` from build metadata when omitted. + +## Interceptors + +**Subscribe** (`SubInterceptor`): wrap the handler invocation. Attach per registration via `WithSubInterceptors` or `RegisterOpt`. + +**Publish** (`PubInterceptor`): wrap the publish call. Pass on `Client.Publish` / generated `Publisher.Interceptors`. + +Both follow a `func(next ...) ...` middleware pattern. + +## Delivery errors + +Return these from handlers to control JetStream acknowledgment: + +| Function | Behavior | +|----------|----------| +| `Reject(errs...)` | Ack and discard; no further retries | +| `Redelivery(delay, errs...)` | Nak with delay (scheduled redelivery) | +| `ForceRetry(errs...)` | Immediate nak / force redelivery | + +Ordinary returned errors trigger the configured retry/backoff policy until `max_retries` is exhausted. + +## Context + +`GetContext(ctx)` returns delivery metadata: subject, stream, consumer, delivery counts, timestamp, HTTP-style headers, and resolved `JobEventConfig`. Replaces the old `GetEventContext` name. + +## Migration from pre-v2 funk cloudevent + +Breaking changes when moving from the previous funk layout: + +| Before | After | +|--------|-------| +| `proto/cloudevent/options.proto` with extensions | `proto/cloudeventoption/options.proto` (`cloudeventoptionpb.E_Job`, `E_Subject`) | +| `GetEventContext` | `GetContext` | +| `PushEvent` / `PushRpcEvent` helpers | Generated `*Publisher.Push*Event` or `Publish` | +| `Publish` returning `(ack, error)` | `Publish` → `result.Result[*PubAckInfo]` | +| `RegisterJobHandler` without opts | `RegisterJobHandler(..., opts ...RegisterOpt)` | +| Single proto package | Split `cloudeventpb` + `cloudeventoptionpb` | + +Checklist for downstream services: + +1. Update proto imports to `cloudeventoption/options.proto`. +2. Regenerate with `protoc-gen-go-cloudevent2` (not `protoc-gen-go-cloudevent`). +3. Replace handler registration with generated `Register*CloudEvent` or pass `RegisterOpt`. +4. Replace publish calls with `*Publisher` or `Publish` + `result` unwrapping. +5. Rename `GetEventContext` → `GetContext`. +6. Align YAML job/subject names with proto `job` / `subject` options. + +## References + +- [CloudEvents Go SDK](https://github.com/cloudevents/sdk-go) (design inspiration) +- [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) diff --git a/component/cloudevent/README.zh.md b/component/cloudevent/README.zh.md new file mode 100644 index 00000000..235edd5b --- /dev/null +++ b/component/cloudevent/README.zh.md @@ -0,0 +1,192 @@ +# CloudEvent 模块 + +基于 NATS JetStream 的 protobuf CloudEvent 任务运行时。通过 `.proto` 中的 service/method 扩展声明 job 与 subject，并由 `protoc-gen-go-cloudevent2` 生成注册与发布代码。 + +## 功能 + +- **JetStream 任务**：通过 YAML 配置 stream、consumer、重试与并发 +- **Protobuf 驱动**：job 名与 subject/topic 来自 proto 扩展 +- **代码生成**：`Register*CloudEvent`、`*Publisher` 及 subject/job 常量 +- **拦截器**：订阅端与发布端中间件链 +- **投递控制**：`Reject`、`Redelivery`、`ForceRetry` +- **类型安全结果**：`Publish` 返回 `result.Result[*PubAckInfo]` + +## 安装 + +```bash +go get github.com/pubgo/funk/v2/component/cloudevent +go install github.com/pubgo/funk/v2/cmds/protoc-gen-go-cloudevent2@latest +``` + +在本仓库根目录： + +```bash +make protobuf +``` + +## Protobuf 结构 + +CloudEvent 刻意拆成 **两个** proto 包： + +| 包 | Go import | 用途 | +|----|-----------|------| +| `proto/cloudevent` | `github.com/pubgo/funk/v2/proto/cloudevent`（`cloudeventpb`） | 消息类型：`PushEventOptions`、`RegisterJobOptions` 等 | +| `proto/cloudeventoption` | `github.com/pubgo/funk/v2/proto/cloudeventoption`（`cloudeventoptionpb`） | 描述符扩展 `E_Job`、`E_Subject` | + +消息类型与扩展分离，业务 proto 只需 import 扩展包即可注解 service/method，而不必在每个生成 API 里混入扩展定义。 + +### 注解示例 + +```protobuf +syntax = "proto3"; + +import "cloudeventoption/options.proto"; + +service GidInnerService { + option (lava.cloudevent.job) = { name: "gid" }; + + rpc ProxyExec(ProxyExecReq) returns (google.protobuf.Empty) { + option (lava.cloudevent.subject) = { name: "gid.proxy.exec" }; + } +} +``` + +- `job.name` 对应 YAML 中 `consumers` 下的 job 键名。 +- `subject.name` 对应该 consumer 下的 subject 配置项。 + +### 代码生成 + +在 `protoc` 参数中加入（参见根目录 `protobuf.yaml`）： + +```text +--go-cloudevent2_out=paths=source_relative:. +``` + +每个带注解的 service 会生成： + +- `CloudEventJobKey` — job 名常量 +- `CloudEventSubjectKey` — subject/topic 常量 +- `CloudEvent` — 含 `On` 字段的 handler 结构体 +- `RegisterCloudEvent` — 注册非 nil 的 handler +- `Publisher` — `{ Client, Opt, Interceptors }` 及 `PushEvent` 方法 + +## 配置 + +任务在 YAML 中声明（结构见 `config.yaml`）： + +```yaml +jobs: + streams: + gid: + storage: "file" + subjects: ["gid.>"] + consumers: + gid: + - consumer: "test:gid" + stream: "gid" + subjects: "gid.proxy.exec" + job: + timeout: "1m" + max_retries: 10 +``` + +加载配置并绑定 NATS JetStream 连接创建 `Client`。配置中的 subject 在运行时会加上前缀（`DefaultPrefix`，默认 `acj`）。 + +默认项（可按 job 覆盖）： + +| 项 | 默认值 | +|----|--------| +| Handler 超时 | `15s` | +| 最大重试 | `3` | +| 重试间隔 | `1s` | +| Consumer `AckWait` | `5m` | + +## 快速开始 + +### 注册 handler（生成代码） + +```go +RegisterGidInnerServiceCloudEvent(jobCli, GidInnerServiceCloudEvent{ + OnProxyExec: func(ctx context.Context, req *gidpb.ProxyExecReq) error { + evt := cloudevent.GetContext(ctx) + return nil + }, +}) +``` + +### 注册 handler（手动） + +```go +cloudevent.RegisterJobHandler(jobCli, "gid", "gid.proxy.exec", + func(ctx context.Context, req *gidpb.ProxyExecReq) error { return nil }, + cloudevent.ProtoRegisterOpts(registerOpts...), + cloudevent.WithSubInterceptors(myInterceptor), +) +``` + +### 发布（生成 Publisher） + +```go +pub := GidInnerServicePublisher{ + Client: jobCli, + Opt: cloudevent.ProtoPubOpts(defaultPushOpts...), +} +ack := pub.PushProxyExecEvent(ctx, req).Unwrap() +``` + +### 发布（直接调用） + +```go +ack := cloudevent.Publish(jobCli, ctx, subjectKey, req, interceptors, opts...).Unwrap() +``` + +`Publish` 会合并 `PubOpt`、默认 `content_type` 为 `application/json`，未指定时自动填充 `sender`。 + +## 拦截器 + +**订阅端**（`SubInterceptor`）：包装 handler 执行，通过 `WithSubInterceptors` 或 `RegisterOpt` 传入。 + +**发布端**（`PubInterceptor`）：包装发布逻辑，通过 `Publisher.Interceptors` 或 `Publish` 参数传入。 + +均采用 `func(next ...) ...` 中间件形式。 + +## 投递错误 + +Handler 返回以下错误可控制 JetStream 确认行为： + +| 函数 | 行为 | +|------|------| +| `Reject(errs...)` | Ack 并丢弃，不再重试 | +| `Redelivery(delay, errs...)` | Nak 并延迟重投 | +| `ForceRetry(errs...)` | 立即 Nak，强制重投 | + +普通 error 按配置的 retry/backoff 重试，直至达到 `max_retries`。 + +## Context + +`GetContext(ctx)` 返回投递元数据：subject、stream、consumer、投递次数、时间戳、HTTP 风格 header 及 `JobEventConfig`。旧 API `GetEventContext` 已更名为 `GetContext`。 + +## 从旧版 funk cloudevent 迁移 + +| 旧版 | 新版 | +|------|------| +| `proto/cloudevent/options.proto` 内联扩展 | `proto/cloudeventoption/options.proto`（`cloudeventoptionpb`） | +| `GetEventContext` | `GetContext` | +| `PushEvent` / `PushRpcEvent` | 生成的 `*Publisher.Push*Event` 或 `Publish` | +| `Publish` 返回 `(ack, error)` | `result.Result[*PubAckInfo]` | +| 无 `RegisterOpt` | `RegisterJobHandler(..., opts ...RegisterOpt)` | +| 单一 proto 包 | `cloudeventpb` + `cloudeventoptionpb` | + +下游服务迁移清单： + +1. proto import 改为 `cloudeventoption/options.proto` +2. 使用 `protoc-gen-go-cloudevent2` 重新生成 +3. 用 `Register*CloudEvent` 或带 `RegisterOpt` 的手动注册替换旧注册方式 +4. 用 `*Publisher` 或 `Publish` + `result` 替换旧发布 helper +5. `GetEventContext` 改名为 `GetContext` +6. YAML job/subject 与 proto 注解保持一致 + +## 参考 + +- [CloudEvents Go SDK](https://github.com/cloudevents/sdk-go) +- [NATS JetStream](https://docs.nats.io/nats-concepts/jetstream) diff --git a/component/cloudevent/_doc.go b/component/cloudevent/_doc.go index 326cb350..2c49e1fc 100644 --- a/component/cloudevent/_doc.go +++ b/component/cloudevent/_doc.go @@ -1,11 +1,8 @@ package cloudevent -// Promise/Future, Group/Yield iterators, and Go helpers for async work. +// NATS JetStream-backed CloudEvent job runtime with protobuf-driven handlers. // -// Protobuf extensions live in github.com/pubgo/funk/v2/proto/cloudeventoption. -// Message types live in github.com/pubgo/funk/v2/proto/cloudevent. +// See README.md for configuration, proto annotations, codegen, and migration notes. // -// Iterator.Await drains the value channel to completion before returning the -// first error, so callers never leave Group workers blocked on send. -// -// https://github.com/octu0/chanque +// Message types: github.com/pubgo/funk/v2/proto/cloudevent +// Descriptor extensions: github.com/pubgo/funk/v2/proto/cloudeventoption From 6fbe40153889cce58b171c9371671ec22325a26d Mon Sep 17 00:00:00 2001 From: barry Date: Wed, 17 Jun 2026 00:09:34 +0800 Subject: [PATCH 3/7] feat(cloudevent): add runnable example and godoc samples Include demo proto/codegen, NATS wiring, and package-level examples documented in README. Co-authored-by: Cursor --- component/cloudevent/README.md | 17 ++ component/cloudevent/README.zh.md | 17 ++ component/cloudevent/example/config.yaml | 13 ++ .../example/demopb/demo.cloudevent.pb.go | 40 +++++ .../cloudevent/example/demopb/demo.pb.go | 146 ++++++++++++++++++ .../cloudevent/example/demopb/demo.proto | 20 +++ .../cloudevent/example/demopb/example_test.go | 13 ++ component/cloudevent/example/main.go | 119 ++++++++++++++ component/cloudevent/example_test.go | 29 ++++ 9 files changed, 414 insertions(+) create mode 100644 component/cloudevent/example/config.yaml create mode 100644 component/cloudevent/example/demopb/demo.cloudevent.pb.go create mode 100644 component/cloudevent/example/demopb/demo.pb.go create mode 100644 component/cloudevent/example/demopb/demo.proto create mode 100644 component/cloudevent/example/demopb/example_test.go create mode 100644 component/cloudevent/example/main.go create mode 100644 component/cloudevent/example_test.go diff --git a/component/cloudevent/README.md b/component/cloudevent/README.md index f44616ce..85524f38 100644 --- a/component/cloudevent/README.md +++ b/component/cloudevent/README.md @@ -189,6 +189,23 @@ Checklist for downstream services: 5. Rename `GetEventContext` → `GetContext`. 6. Align YAML job/subject names with proto `job` / `subject` options. +## Example + +Runnable end-to-end demo (NATS + JetStream required): + +```bash +docker run --rm -p 4222:4222 nats:latest -js +go run ./component/cloudevent/example +``` + +Source layout: + +- `example/main.go` — wires NATS, YAML config, generated register/publish APIs +- `example/demopb/demo.proto` — sample service annotations +- `example/config.yaml` — stream/consumer config matching the proto subjects + +Package-level godoc examples live in `example_test.go`. + ## References - [CloudEvents Go SDK](https://github.com/cloudevents/sdk-go) (design inspiration) diff --git a/component/cloudevent/README.zh.md b/component/cloudevent/README.zh.md index 235edd5b..4b3df648 100644 --- a/component/cloudevent/README.zh.md +++ b/component/cloudevent/README.zh.md @@ -186,6 +186,23 @@ Handler 返回以下错误可控制 JetStream 确认行为： 5. `GetEventContext` 改名为 `GetContext` 6. YAML job/subject 与 proto 注解保持一致 +## 示例 + +可运行的端到端示例（需要 NATS + JetStream）： + +```bash +docker run --rm -p 4222:4222 nats:latest -js +go run ./component/cloudevent/example +``` + +目录说明： + +- `example/main.go` — 连接 NATS、加载 YAML、注册 handler 并发布事件 +- `example/demopb/demo.proto` — 带 cloudevent 注解的示例 service +- `example/config.yaml` — 与 proto subject 对应的 stream/consumer 配置 + +包级 godoc 示例见 `example_test.go`。 + ## 参考 - [CloudEvents Go SDK](https://github.com/cloudevents/sdk-go) diff --git a/component/cloudevent/example/config.yaml b/component/cloudevent/example/config.yaml new file mode 100644 index 00000000..c80a6dbf --- /dev/null +++ b/component/cloudevent/example/config.yaml @@ -0,0 +1,13 @@ +streams: + demo: + storage: memory + subjects: + - demo.> +consumers: + demo: + - stream: demo + concurrent: 1 + subjects: demo.hello.exec + job: + timeout: 30s + max_retries: 3 diff --git a/component/cloudevent/example/demopb/demo.cloudevent.pb.go b/component/cloudevent/example/demopb/demo.cloudevent.pb.go new file mode 100644 index 00000000..c3ce2ce7 --- /dev/null +++ b/component/cloudevent/example/demopb/demo.cloudevent.pb.go @@ -0,0 +1,40 @@ +// Code generated by protoc-gen-go-cloudevent. DO NOT EDIT. +// Versions: +// - protoc-gen-go-cloudevent v0.0.5 +// - protoc v6.33.4 +// source: demo.proto + +package demopb + +import ( + "context" + v2 "github.com/pubgo/funk/v2" + cloudevent "github.com/pubgo/funk/v2/component/cloudevent" + result "github.com/pubgo/funk/v2/result" +) + +const DemoCloudEventJobKey = "demo" + +// HelloExecCloudEventSubjectKey /demo.cloudevent.v1.DemoInnerService/HelloExec +const HelloExecCloudEventSubjectKey = "demo.hello.exec" + +type DemoInnerServiceCloudEvent struct { + OnHelloExec func(ctx context.Context, req *HelloExecReq) error +} + +func RegisterDemoInnerServiceCloudEvent(jobCli *cloudevent.Client, event DemoInnerServiceCloudEvent, opts ...cloudevent.RegisterOpt) { + if event.OnHelloExec != nil { + cloudevent.RegisterJobHandler(jobCli, DemoCloudEventJobKey, HelloExecCloudEventSubjectKey, event.OnHelloExec, opts...) + } + +} + +type DemoInnerServiceCloudEventPublisher struct { + Client *cloudevent.Client + Opt cloudevent.PubOpt + Interceptors []cloudevent.PubInterceptor +} + +func (a DemoInnerServiceCloudEventPublisher) PushHelloExecEvent(ctx context.Context, req *HelloExecReq, opts ...cloudevent.PubOpt) result.Result[*cloudevent.PubAckInfo] { + return a.Client.Publish(ctx, HelloExecCloudEventSubjectKey, req, a.Interceptors, v2.AppendOf(a.Opt, opts...)...) +} diff --git a/component/cloudevent/example/demopb/demo.pb.go b/component/cloudevent/example/demopb/demo.pb.go new file mode 100644 index 00000000..fc6ca61c --- /dev/null +++ b/component/cloudevent/example/demopb/demo.pb.go @@ -0,0 +1,146 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.2 +// protoc v6.33.4 +// source: demo.proto + +package demopb + +import ( + _ "github.com/pubgo/funk/v2/proto/cloudeventoption" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type HelloExecReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *HelloExecReq) Reset() { + *x = HelloExecReq{} + mi := &file_demo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HelloExecReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloExecReq) ProtoMessage() {} + +func (x *HelloExecReq) ProtoReflect() protoreflect.Message { + mi := &file_demo_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloExecReq.ProtoReflect.Descriptor instead. +func (*HelloExecReq) Descriptor() ([]byte, []int) { + return file_demo_proto_rawDescGZIP(), []int{0} +} + +func (x *HelloExecReq) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +var File_demo_proto protoreflect.FileDescriptor + +var file_demo_proto_rawDesc = []byte{ + 0x0a, 0x0a, 0x64, 0x65, 0x6d, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x64, 0x65, + 0x6d, 0x6f, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x31, + 0x1a, 0x1e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x22, 0x0a, + 0x0c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x45, 0x78, 0x65, 0x63, 0x52, 0x65, 0x71, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x32, 0x7c, 0x0a, 0x10, 0x44, 0x65, 0x6d, 0x6f, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5c, 0x0a, 0x09, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x45, 0x78, + 0x65, 0x63, 0x12, 0x20, 0x2e, 0x64, 0x65, 0x6d, 0x6f, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x45, 0x78, 0x65, + 0x63, 0x52, 0x65, 0x71, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x15, 0xda, 0xf1, + 0x04, 0x11, 0x0a, 0x0f, 0x64, 0x65, 0x6d, 0x6f, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x1a, 0x0a, 0xd2, 0xf1, 0x04, 0x06, 0x0a, 0x04, 0x64, 0x65, 0x6d, 0x6f, 0x42, + 0x45, 0x5a, 0x43, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x75, + 0x62, 0x67, 0x6f, 0x2f, 0x66, 0x75, 0x6e, 0x6b, 0x2f, 0x76, 0x32, 0x2f, 0x63, 0x6f, 0x6d, 0x70, + 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2f, 0x64, 0x65, 0x6d, 0x6f, 0x70, 0x62, 0x3b, + 0x64, 0x65, 0x6d, 0x6f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_demo_proto_rawDescOnce sync.Once + file_demo_proto_rawDescData = file_demo_proto_rawDesc +) + +func file_demo_proto_rawDescGZIP() []byte { + file_demo_proto_rawDescOnce.Do(func() { + file_demo_proto_rawDescData = protoimpl.X.CompressGZIP(file_demo_proto_rawDescData) + }) + return file_demo_proto_rawDescData +} + +var file_demo_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_demo_proto_goTypes = []any{ + (*HelloExecReq)(nil), // 0: demo.cloudevent.v1.HelloExecReq + (*emptypb.Empty)(nil), // 1: google.protobuf.Empty +} +var file_demo_proto_depIdxs = []int32{ + 0, // 0: demo.cloudevent.v1.DemoInnerService.HelloExec:input_type -> demo.cloudevent.v1.HelloExecReq + 1, // 1: demo.cloudevent.v1.DemoInnerService.HelloExec:output_type -> google.protobuf.Empty + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_demo_proto_init() } +func file_demo_proto_init() { + if File_demo_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_demo_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_demo_proto_goTypes, + DependencyIndexes: file_demo_proto_depIdxs, + MessageInfos: file_demo_proto_msgTypes, + }.Build() + File_demo_proto = out.File + file_demo_proto_rawDesc = nil + file_demo_proto_goTypes = nil + file_demo_proto_depIdxs = nil +} diff --git a/component/cloudevent/example/demopb/demo.proto b/component/cloudevent/example/demopb/demo.proto new file mode 100644 index 00000000..97d24c1e --- /dev/null +++ b/component/cloudevent/example/demopb/demo.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package demo.cloudevent.v1; + +import "cloudeventoption/options.proto"; +import "google/protobuf/empty.proto"; + +option go_package = "github.com/pubgo/funk/v2/component/cloudevent/example/demopb;demopb"; + +message HelloExecReq { + string name = 1; +} + +service DemoInnerService { + option (lava.cloudevent.job) = {name: "demo"}; + + rpc HelloExec(HelloExecReq) returns (google.protobuf.Empty) { + option (lava.cloudevent.subject) = {name: "demo.hello.exec"}; + } +} diff --git a/component/cloudevent/example/demopb/example_test.go b/component/cloudevent/example/demopb/example_test.go new file mode 100644 index 00000000..91aaf990 --- /dev/null +++ b/component/cloudevent/example/demopb/example_test.go @@ -0,0 +1,13 @@ +package demopb_test + +import ( + "fmt" + + "github.com/pubgo/funk/v2/component/cloudevent/example/demopb" +) + +func ExampleHelloExecCloudEventSubjectKey() { + fmt.Println(demopb.HelloExecCloudEventSubjectKey) + + // Output: demo.hello.exec +} diff --git a/component/cloudevent/example/main.go b/component/cloudevent/example/main.go new file mode 100644 index 00000000..2439a8c6 --- /dev/null +++ b/component/cloudevent/example/main.go @@ -0,0 +1,119 @@ +// Runnable CloudEvent demo. Requires a local NATS server with JetStream enabled. +// +// Start NATS (for example with Docker): +// +// docker run --rm -p 4222:4222 nats:latest -js +// +// Run: +// +// go run ./component/cloudevent/example +// +// Override NATS URL: +// +// NATS_URL=nats://127.0.0.1:4222 go run ./component/cloudevent/example +package main + +import ( + "context" + _ "embed" + "fmt" + "log" + "os" + "time" + + "github.com/pubgo/funk/v2/component/cloudevent" + "github.com/pubgo/funk/v2/component/cloudevent/example/demopb" + "github.com/pubgo/funk/v2/component/lifecycle" + "github.com/pubgo/funk/v2/component/natsclient" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + "github.com/samber/lo" + yaml "gopkg.in/yaml.v3" +) + +//go:embed config.yaml +var configYAML []byte + +type noopLifecycle struct { + beforeStops []lifecycle.ExecFunc +} + +func (l *noopLifecycle) AfterStop(lifecycle.ExecFunc) {} +func (l *noopLifecycle) BeforeStop(f lifecycle.ExecFunc) { l.beforeStops = append(l.beforeStops, f) } +func (l *noopLifecycle) AfterStart(lifecycle.ExecFunc) {} +func (l *noopLifecycle) BeforeStart(lifecycle.ExecFunc) {} + +func (l *noopLifecycle) shutdown() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for i := len(l.beforeStops) - 1; i >= 0; i-- { + _ = l.beforeStops[i](ctx) + } +} + +func loadConfig() (*cloudevent.Config, error) { + cfg := new(cloudevent.Config) + if err := yaml.Unmarshal(configYAML, cfg); err != nil { + return nil, fmt.Errorf("parse config: %w", err) + } + return cfg, nil +} + +func main() { + cfg, err := loadConfig() + if err != nil { + log.Fatal(err) + } + + natsURL := os.Getenv("NATS_URL") + if natsURL == "" { + natsURL = "nats://127.0.0.1:4222" + } + + lc := new(noopLifecycle) + nc := natsclient.New(natsclient.Param{ + Cfg: &natsclient.Config{Url: natsURL}, + Lc: lc, + }) + jobCli := cloudevent.New(cloudevent.Params{ + Nc: nc, + Cfg: cfg, + Lc: lc, + }) + + done := make(chan string, 1) + demopb.RegisterDemoInnerServiceCloudEvent(jobCli, demopb.DemoInnerServiceCloudEvent{ + OnHelloExec: func(ctx context.Context, req *demopb.HelloExecReq) error { + evt := cloudevent.GetContext(ctx) + msg := fmt.Sprintf("hello %s on %s", req.GetName(), evt.Subject) + done <- msg + return nil + }, + }) + + if err := jobCli.Start(); err != nil { + log.Fatal(err) + } + defer lc.shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pub := demopb.DemoInnerServiceCloudEventPublisher{ + Client: jobCli, + Opt: func(po *cloudeventpb.PushEventOptions) { + po.Sender = lo.ToPtr("cloudevent-example") + }, + } + if ack := pub.PushHelloExecEvent(ctx, &demopb.HelloExecReq{Name: "world"}); ack.IsErr() { + log.Fatal(ack.Err()) + } else if info, ok := ack.TryUnwrap(); ok { + log.Printf("published seq=%d", info.AckInfo.Sequence) + } + + select { + case msg := <-done: + log.Println(msg) + case <-ctx.Done(): + log.Fatal("timeout waiting for handler") + } +} diff --git a/component/cloudevent/example_test.go b/component/cloudevent/example_test.go new file mode 100644 index 00000000..977100c3 --- /dev/null +++ b/component/cloudevent/example_test.go @@ -0,0 +1,29 @@ +package cloudevent_test + +import ( + "fmt" + + "github.com/pubgo/funk/v2/component/cloudevent" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + "github.com/samber/lo" +) + +func ExampleWithPushOpt() { + opt := cloudevent.WithPushOpt(func(o *cloudeventpb.PushEventOptions) { + o.ContentType = lo.ToPtr("application/protobuf") + }) + fmt.Println(opt.GetContentType()) + + // Output: application/protobuf +} + +func ExampleProtoRegisterOpts() { + opt := cloudevent.ProtoRegisterOpts(&cloudeventpb.RegisterJobOptions{ + JobName: lo.ToPtr("demo"), + }) + ro := &cloudevent.RegisterJobOptions{Opts: new(cloudeventpb.RegisterJobOptions)} + opt(ro) + fmt.Println(lo.FromPtr(ro.Opts.JobName)) + + // Output: demo +} From 72ddd70dde6ac38b77f9ca311090ff3a4e59d6bf Mon Sep 17 00:00:00 2001 From: barry Date: Wed, 17 Jun 2026 06:17:09 +0800 Subject: [PATCH 4/7] fix(cloudevent): address PR review stability and codegen notes Add panic recovery, release worker pools on stop, guard nil publish opts, derive AckWait from job timeouts, and clean up jennifer output. Co-authored-by: Cursor --- .../protoc-gen-go-cloudevent2/internal/gen.go | 6 +++--- component/cloudevent/client.go | 2 +- component/cloudevent/config.go | 20 +++++++++++++++++++ component/cloudevent/publisher.go | 4 ++++ component/cloudevent/subscriber.go | 6 ++++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/cmds/protoc-gen-go-cloudevent2/internal/gen.go b/cmds/protoc-gen-go-cloudevent2/internal/gen.go index e63aed34..84994c6e 100644 --- a/cmds/protoc-gen-go-cloudevent2/internal/gen.go +++ b/cmds/protoc-gen-go-cloudevent2/internal/gen.go @@ -179,8 +179,8 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated Id(mthName). Params( jen.Id("ctx").Qual("context", "Context"), - jen.Id("req *").Add(getPkg(file, ss.mth.Input.GoIdent)), - jen.Id("opts ...").Qual(cloudeventPkg, "PubOpt"), + jen.Id("req").Op("*").Add(getPkg(file, ss.mth.Input.GoIdent)), + jen.Id("opts").Op("...").Qual(cloudeventPkg, "PubOpt"), ). Params(jen.Qual(resultTypesPkg, "Result").Id("[").Op("*").Qual(cloudeventPkg, "PubAckInfo").Id("]")). BlockFunc(func(group *jen.Group) { @@ -189,7 +189,7 @@ func GenerateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated jen.Id(keyName), jen.Id("req"), jen.Id("a.Interceptors"), - jen.Qual(funkPkg, "AppendOf").Id("(a.Opt, opts...)..."), + jen.Qual(funkPkg, "AppendOf").Call(jen.Id("a.Opt"), jen.Id("opts").Op("...")).Op("..."), ) }) } diff --git a/component/cloudevent/client.go b/component/cloudevent/client.go index d5d83da6..3f284b0b 100644 --- a/component/cloudevent/client.go +++ b/component/cloudevent/client.go @@ -119,7 +119,7 @@ func (c *Client) initConsumer() (r result.Error) { Name: consumerName, Durable: consumerName, Metadata: metadata, - AckWait: time.Minute * 5, + AckWait: consumerAckWait(cfg), } consumer, err := c.js.CreateOrUpdateConsumer(ctx, streamName, consumerCfg) diff --git a/component/cloudevent/config.go b/component/cloudevent/config.go index 59d717c0..9951e53c 100644 --- a/component/cloudevent/config.go +++ b/component/cloudevent/config.go @@ -8,6 +8,7 @@ import ( "github.com/pubgo/funk/v2/buildinfo/version" "github.com/pubgo/funk/v2/errors" "github.com/pubgo/funk/v2/typex" + "github.com/samber/lo" yaml "gopkg.in/yaml.v3" ) @@ -58,6 +59,25 @@ type jobEventHandler struct { interceptors []SubInterceptor } +func consumerAckWait(cfg *ConsumerConfig) time.Duration { + const minAckWait = 5 * time.Minute + + base := handleDefaultJobConfig(cfg.Job) + maxTimeout := lo.FromPtr(base.Timeout) + for _, sub := range cfg.Subjects { + subCfg := mergeJobConfig(lo.ToPtr(JobEventConfig(lo.FromPtr(sub))), base) + if t := lo.FromPtr(subCfg.Timeout); t > maxTimeout { + maxTimeout = t + } + } + + ackWait := maxTimeout + time.Minute + if ackWait < minAckWait { + return minAckWait + } + return ackWait +} + type strOrJobConfig JobEventConfig func (p *strOrJobConfig) UnmarshalYAML(value *yaml.Node) error { diff --git a/component/cloudevent/publisher.go b/component/cloudevent/publisher.go index f978c0ed..b313653a 100644 --- a/component/cloudevent/publisher.go +++ b/component/cloudevent/publisher.go @@ -28,6 +28,10 @@ func (c *Client) Publish(ctx context.Context, topic string, args proto.Message, func (c *Client) doPublish(ctx context.Context, topic string, args proto.Message, opts *PubOptions) (r result.Result[*PubAckInfo]) { defer result.Recovery(&r) + if opts == nil { + opts = new(PubOptions) + } + msgId := xid.New().String() if opts.MsgId != nil { msgId = opts.GetMsgId() diff --git a/component/cloudevent/subscriber.go b/component/cloudevent/subscriber.go index c0301970..8bbaf145 100644 --- a/component/cloudevent/subscriber.go +++ b/component/cloudevent/subscriber.go @@ -8,6 +8,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/panjf2000/ants/v2" "github.com/pubgo/funk/v2/assert" + "github.com/pubgo/funk/v2/component/lifecycle" "github.com/pubgo/funk/v2/errors" "github.com/pubgo/funk/v2/log" "github.com/pubgo/funk/v2/result" @@ -35,6 +36,7 @@ func (c *Client) doConsumeHandler(streamName, consumerName string, jobSubjects m logger.Debug().Func(addMsgInfo).Msg("received cloud job manager") handlerDelayJob := func() (r result.Result[bool]) { + defer result.Recovery(&r) dur := decodeDelayTime(msg.Headers().Get(DelayHeaderKey)). MapErr(func(err error) error { return errors.Wrap(err, "failed to parse job delay time") @@ -141,6 +143,9 @@ func (c *Client) doConsumeHandler(streamName, consumerName string, jobSubjects m ants.WithLogger(log.NewStd(logger)), ants.WithNonblocking(false), )) + c.p.Lc.BeforeStop(lifecycle.WrapNoCtxErr(func() { + pool.Release() + })) return func(msg jetstream.Msg) { if pool.Running() == concurrent { logger.Warn().Func(func(e *zerolog.Event) { @@ -170,6 +175,7 @@ func (c *Client) doErrHandler(streamName, consumerName string) jetstream.PullCon } func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { + defer result.Recovery(&gErr) timeout := lo.FromPtr(cfg.Timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() From 168c0c97f981edfdc2537b298e81fd6ad8855642 Mon Sep 17 00:00:00 2001 From: barry Date: Wed, 17 Jun 2026 06:18:09 +0800 Subject: [PATCH 5/7] Update component/cloudevent/subscriber.go Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- component/cloudevent/subscriber.go | 1 + 1 file changed, 1 insertion(+) diff --git a/component/cloudevent/subscriber.go b/component/cloudevent/subscriber.go index 8bbaf145..f04c86fc 100644 --- a/component/cloudevent/subscriber.go +++ b/component/cloudevent/subscriber.go @@ -174,6 +174,7 @@ func (c *Client) doErrHandler(streamName, consumerName string) jetstream.PullCon }) } +func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { defer result.Recovery(&gErr) timeout := lo.FromPtr(cfg.Timeout) From 4421f97c3124971dce0217aaba8a2a3ae03c9d8f Mon Sep 17 00:00:00 2001 From: barry Date: Wed, 17 Jun 2026 06:21:27 +0800 Subject: [PATCH 6/7] fix(cloudevent): remove duplicate doHandler signature from bad merge GitHub suggestion commit duplicated the function declaration and broke typecheck/lint CI. Co-authored-by: Cursor --- component/cloudevent/subscriber.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/component/cloudevent/subscriber.go b/component/cloudevent/subscriber.go index f04c86fc..3c148cd5 100644 --- a/component/cloudevent/subscriber.go +++ b/component/cloudevent/subscriber.go @@ -7,6 +7,11 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/panjf2000/ants/v2" + "github.com/rs/zerolog" + "github.com/samber/lo" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/component/lifecycle" "github.com/pubgo/funk/v2/errors" @@ -14,10 +19,6 @@ import ( "github.com/pubgo/funk/v2/result" "github.com/pubgo/funk/v2/stack" "github.com/pubgo/funk/v2/try" - "github.com/rs/zerolog" - "github.com/samber/lo" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" ) func (c *Client) doConsumeHandler(streamName, consumerName string, jobSubjects map[string]*jobEventHandler, concurrent int) func(msg jetstream.Msg) { @@ -174,7 +175,6 @@ func (c *Client) doErrHandler(streamName, consumerName string) jetstream.PullCon }) } -func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { func (c *Client) doHandler(meta *jetstream.MsgMetadata, msg jetstream.Msg, job *jobEventHandler, cfg *JobEventConfig) (gErr result.Error) { defer result.Recovery(&gErr) timeout := lo.FromPtr(cfg.Timeout) From d569c3cd4cb5ddd6d64ee79a8f61e6718d0c40b1 Mon Sep 17 00:00:00 2001 From: barry Date: Wed, 17 Jun 2026 06:22:35 +0800 Subject: [PATCH 7/7] style(cloudevent): normalize imports for gci lint Apply golangci-lint import grouping across the cloudevent package. Co-authored-by: Cursor --- component/cloudevent/aaa.go | 7 ++++--- component/cloudevent/client.go | 17 +++++++++-------- component/cloudevent/config.go | 15 ++++++++------- component/cloudevent/context.go | 3 ++- component/cloudevent/example/main.go | 11 ++++++----- component/cloudevent/example_test.go | 3 ++- component/cloudevent/publisher.go | 9 +++++---- component/cloudevent/util.go | 13 +++++++------ 8 files changed, 43 insertions(+), 35 deletions(-) diff --git a/component/cloudevent/aaa.go b/component/cloudevent/aaa.go index af935e2d..c0c3b9bc 100644 --- a/component/cloudevent/aaa.go +++ b/component/cloudevent/aaa.go @@ -5,11 +5,12 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" - "github.com/pubgo/funk/v2/log" - "github.com/pubgo/funk/v2/result" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" + + "github.com/pubgo/funk/v2/log" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + "github.com/pubgo/funk/v2/result" ) var logger = log.GetLogger("cloudevent") diff --git a/component/cloudevent/client.go b/component/cloudevent/client.go index 3f284b0b..05379501 100644 --- a/component/cloudevent/client.go +++ b/component/cloudevent/client.go @@ -8,18 +8,19 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/samber/lo" + "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/buildinfo/version" "github.com/pubgo/funk/v2/component/lifecycle" "github.com/pubgo/funk/v2/component/natsclient" "github.com/pubgo/funk/v2/errors" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" "github.com/pubgo/funk/v2/result" "github.com/pubgo/funk/v2/running" "github.com/pubgo/funk/v2/stack" "github.com/pubgo/funk/v2/typex" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" - "github.com/rs/zerolog" - "github.com/samber/lo" ) type Params struct { @@ -46,12 +47,12 @@ type Client struct { p Params js jetstream.JetStream - streams map[string]jetstream.Stream - consumers map[string]map[string]*Consumer + streams map[string]jetstream.Stream + consumers map[string]map[string]*Consumer jobManagers map[string]*jobManager - jobs map[string]map[string]map[string]*jobEventHandler - prefix string - subjects map[string]*cloudeventpb.CloudEventMethodOptions + jobs map[string]map[string]map[string]*jobEventHandler + prefix string + subjects map[string]*cloudeventpb.CloudEventMethodOptions } func (c *Client) initStream() (r result.Error) { diff --git a/component/cloudevent/config.go b/component/cloudevent/config.go index 9951e53c..c229e5c4 100644 --- a/component/cloudevent/config.go +++ b/component/cloudevent/config.go @@ -4,12 +4,13 @@ import ( "fmt" "time" + "github.com/samber/lo" + yaml "gopkg.in/yaml.v3" + "github.com/pubgo/funk/v2/assert" "github.com/pubgo/funk/v2/buildinfo/version" "github.com/pubgo/funk/v2/errors" "github.com/pubgo/funk/v2/typex" - "github.com/samber/lo" - yaml "gopkg.in/yaml.v3" ) const ( @@ -28,7 +29,7 @@ const ( var senderValue = fmt.Sprintf("%s/%s", version.Project(), version.Version()) type Config struct { - Streams map[string]*StreamConfig `yaml:"streams"` + Streams map[string]*StreamConfig `yaml:"streams"` Consumers map[string]typex.YamlListType[*ConsumerConfig] `yaml:"consumers"` } @@ -38,11 +39,11 @@ type StreamConfig struct { } type ConsumerConfig struct { - Consumer *string `yaml:"consumer"` - Concurrent *int `yaml:"concurrent"` - Stream string `yaml:"stream"` + Consumer *string `yaml:"consumer"` + Concurrent *int `yaml:"concurrent"` + Stream string `yaml:"stream"` Subjects typex.YamlListType[*strOrJobConfig] `yaml:"subjects"` - Job *JobEventConfig `yaml:"job"` + Job *JobEventConfig `yaml:"job"` } type JobEventConfig struct { diff --git a/component/cloudevent/context.go b/component/cloudevent/context.go index 232abdb7..76c01bb3 100644 --- a/component/cloudevent/context.go +++ b/component/cloudevent/context.go @@ -5,10 +5,11 @@ import ( "net/http" "time" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" "github.com/rs/xid" "github.com/samber/lo" "google.golang.org/protobuf/proto" + + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" ) type Context struct { diff --git a/component/cloudevent/example/main.go b/component/cloudevent/example/main.go index 2439a8c6..270c79f9 100644 --- a/component/cloudevent/example/main.go +++ b/component/cloudevent/example/main.go @@ -21,13 +21,14 @@ import ( "os" "time" + "github.com/samber/lo" + yaml "gopkg.in/yaml.v3" + "github.com/pubgo/funk/v2/component/cloudevent" "github.com/pubgo/funk/v2/component/cloudevent/example/demopb" "github.com/pubgo/funk/v2/component/lifecycle" "github.com/pubgo/funk/v2/component/natsclient" cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" - "github.com/samber/lo" - yaml "gopkg.in/yaml.v3" ) //go:embed config.yaml @@ -37,10 +38,10 @@ type noopLifecycle struct { beforeStops []lifecycle.ExecFunc } -func (l *noopLifecycle) AfterStop(lifecycle.ExecFunc) {} +func (l *noopLifecycle) AfterStop(lifecycle.ExecFunc) {} func (l *noopLifecycle) BeforeStop(f lifecycle.ExecFunc) { l.beforeStops = append(l.beforeStops, f) } -func (l *noopLifecycle) AfterStart(lifecycle.ExecFunc) {} -func (l *noopLifecycle) BeforeStart(lifecycle.ExecFunc) {} +func (l *noopLifecycle) AfterStart(lifecycle.ExecFunc) {} +func (l *noopLifecycle) BeforeStart(lifecycle.ExecFunc) {} func (l *noopLifecycle) shutdown() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/component/cloudevent/example_test.go b/component/cloudevent/example_test.go index 977100c3..30b07369 100644 --- a/component/cloudevent/example_test.go +++ b/component/cloudevent/example_test.go @@ -3,9 +3,10 @@ package cloudevent_test import ( "fmt" + "github.com/samber/lo" + "github.com/pubgo/funk/v2/component/cloudevent" cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" - "github.com/samber/lo" ) func ExampleWithPushOpt() { diff --git a/component/cloudevent/publisher.go b/component/cloudevent/publisher.go index b313653a..cc5fac5c 100644 --- a/component/cloudevent/publisher.go +++ b/component/cloudevent/publisher.go @@ -6,15 +6,16 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "github.com/pubgo/funk/v2/ctxutil" - "github.com/pubgo/funk/v2/errors" - "github.com/pubgo/funk/v2/result" - "github.com/pubgo/funk/v2/typex" "github.com/rs/xid" "github.com/rs/zerolog" "github.com/samber/lo" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + + "github.com/pubgo/funk/v2/ctxutil" + "github.com/pubgo/funk/v2/errors" + "github.com/pubgo/funk/v2/result" + "github.com/pubgo/funk/v2/typex" ) func Publish(jobCli *Client, ctx context.Context, topic string, args proto.Message, interceptors []PubInterceptor, opts ...PubOpt) result.Result[*PubAckInfo] { diff --git a/component/cloudevent/util.go b/component/cloudevent/util.go index d8cef89b..0b74cfde 100644 --- a/component/cloudevent/util.go +++ b/component/cloudevent/util.go @@ -7,16 +7,17 @@ import ( "time" "github.com/nats-io/nats.go/jetstream" - "github.com/pubgo/funk/v2/assert" - "github.com/pubgo/funk/v2/errors" - "github.com/pubgo/funk/v2/protoutils" - "github.com/pubgo/funk/v2/result" - cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" - cloudeventoptionpb "github.com/pubgo/funk/v2/proto/cloudeventoption" "github.com/rs/zerolog" "github.com/samber/lo" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/known/durationpb" + + "github.com/pubgo/funk/v2/assert" + "github.com/pubgo/funk/v2/errors" + cloudeventpb "github.com/pubgo/funk/v2/proto/cloudevent" + cloudeventoptionpb "github.com/pubgo/funk/v2/proto/cloudeventoption" + "github.com/pubgo/funk/v2/protoutils" + "github.com/pubgo/funk/v2/result" ) func getStorageType(name string) jetstream.StorageType {