From 60a1276c50ac7e2cb06b426596593b482c0165d8 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Tue, 16 Apr 2024 12:53:56 -0600 Subject: [PATCH 01/17] feat: gRCP setup --- cmd/provider-services/cmd/grpc.go | 21 +++ cmd/provider-services/cmd/manifest.go | 89 ++++++--- cmd/provider-services/cmd/run.go | 21 ++- gateway/grpc/client.go | 56 ++++++ gateway/grpc/context.go | 41 +++++ gateway/grpc/lease.go | 34 ++++ gateway/grpc/provider.go | 43 +++++ gateway/grpc/server.go | 222 ++++++++--------------- gateway/grpc/server_test.go | 252 ++++++++++++++++++++++++++ gateway/rest/client.go | 73 ++------ gateway/utils/utils.go | 143 +++++++++------ 11 files changed, 699 insertions(+), 296 deletions(-) create mode 100644 cmd/provider-services/cmd/grpc.go create mode 100644 gateway/grpc/client.go create mode 100644 gateway/grpc/context.go create mode 100644 gateway/grpc/lease.go create mode 100644 gateway/grpc/provider.go create mode 100644 gateway/grpc/server_test.go diff --git a/cmd/provider-services/cmd/grpc.go b/cmd/provider-services/cmd/grpc.go new file mode 100644 index 000000000..f88cc8e28 --- /dev/null +++ b/cmd/provider-services/cmd/grpc.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "fmt" + "net" + "net/url" +) + +func grpcURI(hostURI string) (string, error) { + u, err := url.Parse(hostURI) + if err != nil { + return "", fmt.Errorf("url parse: %w", err) + } + + h, _, err := net.SplitHostPort(u.Host) + if err != nil { + return "", fmt.Errorf("split host port: %w", err) + } + + return net.JoinHostPort(h, "8444"), nil +} diff --git a/cmd/provider-services/cmd/manifest.go b/cmd/provider-services/cmd/manifest.go index 56379dfe6..1bac06638 100644 --- a/cmd/provider-services/cmd/manifest.go +++ b/cmd/provider-services/cmd/manifest.go @@ -2,9 +2,11 @@ package cmd import ( "bytes" + "context" "crypto/tls" "encoding/json" "fmt" + "time" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -14,16 +16,17 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" + ptypes "github.com/akash-network/akash-api/go/node/provider/v1beta3" + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" "github.com/akash-network/node/sdl" cutils "github.com/akash-network/node/x/cert/utils" aclient "github.com/akash-network/provider/client" + gwgrpc "github.com/akash-network/provider/gateway/grpc" gwrest "github.com/akash-network/provider/gateway/rest" ) -var ( - errSubmitManifestFailed = errors.New("submit manifest to some providers has been failed") -) +var errSubmitManifestFailed = errors.New("submit manifest to some providers has been failed") // SendManifestCmd looks up the Providers blockchain information, // and POSTs the SDL file to the Gateway address. @@ -94,32 +97,68 @@ func doSendManifest(cmd *cobra.Command, sdlpath string) error { ErrorMessage string `json:"errorMessage,omitempty" yaml:"errorMessage,omitempty"` } - results := make([]result, len(leases)) - - submitFailed := false + var ( + results = make([]result, len(leases)) + submitFailed = false + ) for i, lid := range leases { - prov, _ := sdk.AccAddressFromBech32(lid.Provider) - gclient, err := gwrest.NewClient(cl, prov, []tls.Certificate{cert}) - if err != nil { - return err - } + err = func() error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + provAddr, _ := sdk.AccAddressFromBech32(lid.Provider) + prov, err := cl.Provider(ctx, &ptypes.QueryProviderRequest{Owner: provAddr.String()}) + if err != nil { + return fmt.Errorf("query client provider: %w", err) + } - err = gclient.SubmitManifest(cmd.Context(), dseq, mani) - res := result{ - Provider: prov, - Status: "PASS", - } - if err != nil { - res.Error = err.Error() - if e, valid := err.(gwrest.ClientResponseError); valid { - res.ErrorMessage = e.Message + hostURIgRPC, err := grpcURI(prov.GetProvider().HostURI) + if err != nil { + return fmt.Errorf("grpc uri: %w", err) } - res.Status = "FAIL" - submitFailed = true - } - results[i] = res + res := result{ + Provider: provAddr, + Status: "PASS", + } + + c, err := gwgrpc.NewClient(ctx, hostURIgRPC, cert, cl) + if err == nil { + defer c.Close() + + if _, err = c.SendManifest(ctx, &leasev1.SendManifestRequest{ + LeaseId: lid, + Manifest: mani, + }); err != nil { + res.Error = err.Error() + res.Status = "FAIL" + submitFailed = true + } + } else { + gclient, err := gwrest.NewClient(cl, provAddr, []tls.Certificate{cert}) + if err != nil { + return fmt.Errorf("gwrest new client: %w", err) + } + + err = gclient.SubmitManifest(cmd.Context(), dseq, mani) + if err != nil { + res.Error = err.Error() + if e, valid := err.(gwrest.ClientResponseError); valid { + res.ErrorMessage = e.Message + } + res.Status = "FAIL" + submitFailed = true + } + } + + results[i] = res + + return nil + }() + if err != nil { + return err + } } buf := &bytes.Buffer{} @@ -146,7 +185,6 @@ func doSendManifest(cmd *cobra.Command, sdlpath string) error { } _, err = fmt.Fprint(cmd.OutOrStdout(), buf.String()) - if err != nil { return err } @@ -157,3 +195,4 @@ func doSendManifest(cmd *cobra.Command, sdlpath string) error { return nil } + diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 5eeec75e8..0ed899ae0 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -109,9 +109,7 @@ const ( serviceHostnameOperator = "hostname-operator" ) -var ( - errInvalidConfig = errors.New("Invalid configuration") -) +var errInvalidConfig = errors.New("Invalid configuration") // RunCmd launches the Akash Provider service func RunCmd() *cobra.Command { @@ -201,12 +199,12 @@ func RunCmd() *cobra.Command { panic(err) } - cmd.Flags().String(FlagGatewayListenAddress, "0.0.0.0:8443", "Gateway listen address") + cmd.Flags().String(FlagGatewayListenAddress, "0.0.0.0:8443", "REST gateway listen address") if err := viper.BindPFlag(FlagGatewayListenAddress, cmd.Flags().Lookup(FlagGatewayListenAddress)); err != nil { panic(err) } - cmd.Flags().String(FlagGatewayGRPCListenAddress, "0.0.0.0:8444", "Gateway listen address") + cmd.Flags().String(FlagGatewayGRPCListenAddress, "0.0.0.0:8444", "gRPC gateway listen address") if err := viper.BindPFlag(FlagGatewayGRPCListenAddress, cmd.Flags().Lookup(FlagGatewayGRPCListenAddress)); err != nil { panic(err) } @@ -420,9 +418,11 @@ var allowedBidPricingStrategies = [...]string{ bidPricingStrategyShellScript, } -var errNoSuchBidPricingStrategy = fmt.Errorf("No such bid pricing strategy. Allowed: %v", allowedBidPricingStrategies) -var errInvalidValueForBidPrice = errors.New("not a valid bid price") -var errBidPriceNegative = errors.New("Bid price cannot be a negative number") +var ( + errNoSuchBidPricingStrategy = fmt.Errorf("No such bid pricing strategy. Allowed: %v", allowedBidPricingStrategies) + errInvalidValueForBidPrice = errors.New("not a valid bid price") + errBidPriceNegative = errors.New("Bid price cannot be a negative number") +) func strToBidPriceScale(val string) (decimal.Decimal, error) { v, err := decimal.NewFromString(val) @@ -746,8 +746,9 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { return err } - err = gwgrpc.NewServer(ctx, grpcaddr, []tls.Certificate{tlsCert}, service) - if err != nil { + ctx = gwgrpc.ContextWithQueryClient(ctx, cl.Query()) + + if err = gwgrpc.Serve(ctx, grpcaddr, []tls.Certificate{tlsCert}, service); err != nil { return err } diff --git a/gateway/grpc/client.go b/gateway/grpc/client.go new file mode 100644 index 000000000..e129f557a --- /dev/null +++ b/gateway/grpc/client.go @@ -0,0 +1,56 @@ +package grpc + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" + providerv1 "github.com/akash-network/akash-api/go/provider/v1" + + "github.com/akash-network/provider/gateway/utils" +) + +type Client struct { + providerv1.ProviderRPCClient + leasev1.LeaseRPCClient + + conn *grpc.ClientConn +} + +func (c *Client) Close() error { + return c.conn.Close() +} + +func NewClient(ctx context.Context, addr string, cert tls.Certificate, cquery ctypes.QueryClient) (*Client, error) { + tlsConfig := tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS13, + VerifyPeerCertificate: func(certificates [][]byte, _ [][]*x509.Certificate) error { + if _, err := utils.VerifyOwnerCertBytes(ctx, certificates, "", x509.ExtKeyUsageClientAuth, cquery); err != nil { + return err + } + return nil + }, + } + + conn, err := grpc.DialContext(ctx, addr, + grpc.WithBlock(), + grpc.WithTransportCredentials(credentials.NewTLS(&tlsConfig)), + ) + if err != nil { + return nil, fmt.Errorf("grpc dial context %s: %w", addr, err) + } + + return &Client{ + ProviderRPCClient: providerv1.NewProviderRPCClient(conn), + LeaseRPCClient: leasev1.NewLeaseRPCClient(conn), + + conn: conn, + }, nil +} diff --git a/gateway/grpc/context.go b/gateway/grpc/context.go new file mode 100644 index 000000000..f697c818e --- /dev/null +++ b/gateway/grpc/context.go @@ -0,0 +1,41 @@ +package grpc + +import ( + "context" + + ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +type ContextKey string + +const ( + ContextKeyQueryClient = ContextKey("query-client") + ContextKeyOwner = ContextKey("owner") +) + +func ContextWithQueryClient(ctx context.Context, c ctypes.QueryClient) context.Context { + return context.WithValue(ctx, ContextKeyQueryClient, c) +} + +func MustQueryClientFromCtx(ctx context.Context) ctypes.QueryClient { + val := ctx.Value(ContextKeyQueryClient) + if val == nil { + panic("context does not have query client set") + } + + return val.(ctypes.QueryClient) +} + +func ContextWithOwner(ctx context.Context, address sdk.Address) context.Context { + return context.WithValue(ctx, ContextKeyOwner, address) +} + +func OwnerFromCtx(ctx context.Context) sdk.Address { + val := ctx.Value(ContextKeyOwner) + if val == nil { + return sdk.AccAddress{} + } + + return val.(sdk.Address) +} diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go new file mode 100644 index 000000000..fe15446a6 --- /dev/null +++ b/gateway/grpc/lease.go @@ -0,0 +1,34 @@ +package grpc + +import ( + "context" + + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" + + "github.com/akash-network/provider" +) + +type leaseV1 struct { + c provider.Client + ctx context.Context +} + +func (l *leaseV1) SendManifest(context.Context, *leasev1.SendManifestRequest) (*leasev1.SendManifestResponse, error) { + panic("unimplemented") +} + +func (l *leaseV1) ServiceLogs(context.Context, *leasev1.ServiceLogsRequest) (*leasev1.ServiceLogsResponse, error) { + panic("unimplemented") +} + +func (l *leaseV1) ServiceStatus(context.Context, *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { + panic("unimplemented") +} + +func (l *leaseV1) StreamServiceLogs(*leasev1.ServiceLogsRequest, leasev1.LeaseRPC_StreamServiceLogsServer) error { + panic("unimplemented") +} + +func (l *leaseV1) StreamServiceStatus(*leasev1.ServiceStatusRequest, leasev1.LeaseRPC_StreamServiceStatusServer) error { + panic("unimplemented") +} diff --git a/gateway/grpc/provider.go b/gateway/grpc/provider.go new file mode 100644 index 000000000..7c6cd1d25 --- /dev/null +++ b/gateway/grpc/provider.go @@ -0,0 +1,43 @@ +package grpc + +import ( + providerv1 "github.com/akash-network/akash-api/go/provider/v1" + "golang.org/x/net/context" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/akash-network/provider" + "github.com/akash-network/provider/tools/fromctx" + ptypes "github.com/akash-network/provider/types" +) + +type providerV1 struct { + ctx context.Context + c provider.Client +} + +func (p *providerV1) GetStatus(ctx context.Context, _ *emptypb.Empty) (*providerv1.Status, error) { + return p.c.StatusV1(ctx) +} + +func (p *providerV1) StreamStatus(_ *emptypb.Empty, stream providerv1.ProviderRPC_StreamStatusServer) error { + bus, err := fromctx.PubSubFromCtx(p.ctx) + if err != nil { + return err + } + + events := bus.Sub(ptypes.PubSubTopicProviderStatus) + + for { + select { + case <-p.ctx.Done(): + return p.ctx.Err() + case <-stream.Context().Done(): + return stream.Context().Err() + case evt := <-events: + val := evt.(providerv1.Status) + if err := stream.Send(&val); err != nil { + return err + } + } + } +} diff --git a/gateway/grpc/server.go b/gateway/grpc/server.go index 941f7cc8d..e62c20553 100644 --- a/gateway/grpc/server.go +++ b/gateway/grpc/server.go @@ -1,96 +1,48 @@ package grpc import ( + "context" "crypto/tls" "crypto/x509" - "errors" "fmt" "net" "time" - "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/peer" - "google.golang.org/protobuf/types/known/emptypb" - - sdk "github.com/cosmos/cosmos-sdk/types" "github.com/akash-network/akash-api/go/grpc/gogoreflection" ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" providerv1 "github.com/akash-network/akash-api/go/provider/v1" + cmblog "github.com/tendermint/tendermint/libs/log" "github.com/akash-network/provider" + "github.com/akash-network/provider/gateway/utils" "github.com/akash-network/provider/tools/fromctx" - ptypes "github.com/akash-network/provider/types" ) -type ContextKey string - -const ( - ContextKeyQueryClient = ContextKey("query-client") - ContextKeyOwner = ContextKey("owner") +var ( + _ providerv1.ProviderRPCServer = (*server)(nil) + _ leasev1.LeaseRPCServer = (*server)(nil) ) -type grpcProviderV1 struct { - ctx context.Context - client provider.StatusClient -} - -var _ providerv1.ProviderRPCServer = (*grpcProviderV1)(nil) - -func QueryClientFromCtx(ctx context.Context) ctypes.QueryClient { - val := ctx.Value(ContextKeyQueryClient) - if val == nil { - panic("context does not have pubsub set") - } - - return val.(ctypes.QueryClient) -} - -func ContextWithOwner(ctx context.Context, address sdk.Address) context.Context { - return context.WithValue(ctx, ContextKeyOwner, address) -} - -func OwnerFromCtx(ctx context.Context) sdk.Address { - val := ctx.Value(ContextKeyOwner) - if val == nil { - return sdk.AccAddress{} - } - - return val.(sdk.Address) +type server struct { + *providerV1 + *leaseV1 } -func NewServer(ctx context.Context, endpoint string, certs []tls.Certificate, client provider.StatusClient) error { - // InsecureSkipVerify is set to true due to inability to use normal TLS verification - // certificate validation and authentication performed later in mtlsHandler - tlsConfig := &tls.Config{ - Certificates: certs, - ClientAuth: tls.RequestClientCert, - InsecureSkipVerify: true, // nolint: gosec - MinVersion: tls.VersionTLS13, - } - +func Serve(ctx context.Context, endpoint string, certs []tls.Certificate, c provider.Client) error { group, err := fromctx.ErrGroupFromCtx(ctx) if err != nil { return err } - log := fromctx.LogcFromCtx(ctx) - - grpcSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ - MinTime: 30 * time.Second, - PermitWithoutStream: false, - }), grpc.ChainUnaryInterceptor(mtlsInterceptor())) - - pRPC := &grpcProviderV1{ - ctx: ctx, - client: client, - } + grpcSrv := newServer(ctx, certs, c) - providerv1.RegisterProviderRPCServer(grpcSrv, pRPC) - gogoreflection.Register(grpcSrv) + log := fromctx.LogcFromCtx(ctx) group.Go(func() error { grpcLis, err := net.Listen("tcp", endpoint) @@ -114,102 +66,80 @@ func NewServer(ctx context.Context, endpoint string, certs []tls.Certificate, cl return nil } -func mtlsInterceptor() grpc.UnaryServerInterceptor { - return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { +func newServer(ctx context.Context, certs []tls.Certificate, c provider.Client) *grpc.Server { + // InsecureSkipVerify is set to true due to inability to use normal TLS verification + // certificate validation and authentication performed later in mtlsHandler + tlsConfig := &tls.Config{ + Certificates: certs, + ClientAuth: tls.RequestClientCert, + InsecureSkipVerify: true, // nolint: gosec + MinVersion: tls.VersionTLS13, + } + + cquery := MustQueryClientFromCtx(ctx) + + g := grpc.NewServer( + grpc.Creds( + credentials.NewTLS(tlsConfig), + ), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: 30 * time.Second, + PermitWithoutStream: false, + }), + grpc.ChainUnaryInterceptor( + mtlsInterceptor(cquery), + errorLogInterceptor(fromctx.LogcFromCtx(ctx)), + ), + ) + + s := &server{ + providerV1: &providerV1{ + ctx: ctx, + c: c, + }, + leaseV1: &leaseV1{ + ctx: ctx, + c: c, + }, + } + + providerv1.RegisterProviderRPCServer(g, s) + leasev1.RegisterLeaseRPCServer(g, s) + gogoreflection.Register(g) + + return g +} + +func mtlsInterceptor(cquery ctypes.QueryClient) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { if p, ok := peer.FromContext(ctx); ok { if mtls, ok := p.AuthInfo.(credentials.TLSInfo); ok { - certificates := mtls.State.PeerCertificates - - if len(certificates) > 0 { - if len(certificates) != 1 { - return nil, fmt.Errorf("tls: invalid certificate chain") // nolint: goerr113 - } - - cquery := QueryClientFromCtx(ctx) - - cert := certificates[0] - - // validation - var owner sdk.Address - if owner, err = sdk.AccAddressFromBech32(cert.Subject.CommonName); err != nil { - return nil, fmt.Errorf("tls: invalid certificate's subject common name: %w", err) - } - - // 1. CommonName in issuer and Subject must match and be as Bech32 format - if cert.Subject.CommonName != cert.Issuer.CommonName { - return nil, fmt.Errorf("tls: invalid certificate's issuer common name: %w", err) - } - - // 2. serial number must be in - if cert.SerialNumber == nil { - return nil, fmt.Errorf("tls: invalid certificate serial number: %w", err) - } - - // 3. look up certificate on chain - var resp *ctypes.QueryCertificatesResponse - resp, err = cquery.Certificates( - ctx, - &ctypes.QueryCertificatesRequest{ - Filter: ctypes.CertificateFilter{ - Owner: owner.String(), - Serial: cert.SerialNumber.String(), - State: "valid", - }, - }, - ) - if err != nil { - return nil, fmt.Errorf("tls: unable to fetch certificate from chain: %w", err) - } - if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { - return nil, errors.New("tls: attempt to use non-existing or revoked certificate") // nolint: goerr113 - } - - clientCertPool := x509.NewCertPool() - clientCertPool.AddCert(cert) - - opts := x509.VerifyOptions{ - Roots: clientCertPool, - CurrentTime: time.Now(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - MaxConstraintComparisions: 0, - } - - if _, err = cert.Verify(opts); err != nil { - return nil, fmt.Errorf("tls: unable to verify certificate: %w", err) - } + owner, err := utils.VerifyOwnerCert(ctx, mtls.State.PeerCertificates, "", x509.ExtKeyUsageClientAuth, cquery) + if err != nil { + return nil, fmt.Errorf("verify cert chain: %w", err) + } + if owner != nil { ctx = ContextWithOwner(ctx, owner) } } } - return handler(ctx, req) + return next(ctx, req) } } -func (gm *grpcProviderV1) GetStatus(ctx context.Context, _ *emptypb.Empty) (*providerv1.Status, error) { - return gm.client.StatusV1(ctx) -} - -func (gm *grpcProviderV1) StreamStatus(_ *emptypb.Empty, stream providerv1.ProviderRPC_StreamStatusServer) error { - bus, err := fromctx.PubSubFromCtx(gm.ctx) - if err != nil { - return err - } - - events := bus.Sub(ptypes.PubSubTopicProviderStatus) - - for { - select { - case <-gm.ctx.Done(): - return gm.ctx.Err() - case <-stream.Context().Done(): - return stream.Context().Err() - case evt := <-events: - val := evt.(providerv1.Status) - if err := stream.Send(&val); err != nil { - return err - } +// TODO(andrewhare): Possibly replace this with +// https://github.com/grpc-ecosystem/go-grpc-middleware/tree/main/interceptors/logging +// to get full request/response logging? +func errorLogInterceptor(l cmblog.Logger) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req any, i *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { + resp, err := next(ctx, req) + if err != nil { + l.Error(i.FullMethod, "err", err) } + + return resp, err } } + diff --git a/gateway/grpc/server_test.go b/gateway/grpc/server_test.go new file mode 100644 index 000000000..6780171cf --- /dev/null +++ b/gateway/grpc/server_test.go @@ -0,0 +1,252 @@ +package grpc + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/protobuf/types/known/emptypb" + + types "github.com/akash-network/akash-api/go/node/cert/v1beta3" + qmock "github.com/akash-network/akash-api/go/node/client/v1beta2/mocks" + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" + providerv1 "github.com/akash-network/akash-api/go/provider/v1" + "github.com/akash-network/node/testutil" + + "github.com/akash-network/provider/mocks" +) + +type asserter interface { + AssertExpectations(mock.TestingT) bool +} + +type client struct { + p providerv1.ProviderRPCClient + l leasev1.LeaseRPCClient +} + +func TestRPCs(t *testing.T) { + var ( + qclient = &qmock.QueryClient{} + com = testutil.CertificateOptionMocks(qclient) + cod = testutil.CertificateOptionDomains([]string{"localhost", "127.0.0.1"}) + ) + + var ( + crt1 = testutil.Certificate(t, testutil.AccAddress(t), com, cod) + crt2 = testutil.Certificate(t, testutil.AccAddress(t), com, cod) + ) + + qclient.EXPECT().Certificates(mock.Anything, mock.Anything).Return(&types.QueryCertificatesResponse{ + Certificates: types.CertificatesResponse{ + types.CertificateResponse{ + Certificate: types.Certificate{ + State: types.CertificateValid, + Cert: crt2.PEM.Cert, + Pubkey: crt2.PEM.Pub, + }, + Serial: crt2.Serial.String(), + }, + }, + }, nil) + + cases := []struct { + desc string + mocks func() (*mocks.Client, []asserter) + run func(context.Context, *testing.T, client) + }{ + { + desc: "GetStatus", + mocks: func() (*mocks.Client, []asserter) { + var c mocks.Client + c.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) + return &c, nil + }, + run: func(ctx context.Context, t *testing.T, c client) { + _, err := c.p.GetStatus(ctx, &emptypb.Empty{}) + assert.NoError(t, err) + }, + }, + } + + for _, c := range cases { + c := c + + t.Run(c.desc, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ctx = ContextWithQueryClient(ctx, qclient) + + mc, as := c.mocks() + defer mc.AssertExpectations(t) + + for _, a := range as { + defer a.AssertExpectations(t) + } + + s := newServer(ctx, crt1.Cert, mc) + defer s.Stop() + + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + + go func() { + require.NoError(t, s.Serve(l)) + }() + + tlsConfig := tls.Config{ + InsecureSkipVerify: true, + Certificates: crt2.Cert, + } + + conn, err := grpc.DialContext(ctx, l.Addr().String(), + grpc.WithTransportCredentials(credentials.NewTLS(&tlsConfig))) + require.NoError(t, err) + + defer conn.Close() + + c.run(ctx, t, client{ + p: providerv1.NewProviderRPCClient(conn), + l: leasev1.NewLeaseRPCClient(conn), + }) + }) + } +} + +func TestMTLS(t *testing.T) { + var ( + qclient = &qmock.QueryClient{} + com = testutil.CertificateOptionMocks(qclient) + cod = testutil.CertificateOptionDomains([]string{"localhost", "127.0.0.1"}) + ) + + crt := testutil.Certificate(t, testutil.AccAddress(t), com, cod) + + qclient.EXPECT().Certificates(mock.Anything, mock.Anything).Return(&types.QueryCertificatesResponse{ + Certificates: types.CertificatesResponse{ + types.CertificateResponse{ + Certificate: types.Certificate{ + State: types.CertificateValid, + Cert: crt.PEM.Cert, + Pubkey: crt.PEM.Pub, + }, + Serial: crt.Serial.String(), + }, + }, + }, nil) + + cases := []struct { + desc string + cert func(*testing.T) tls.Certificate + errContains string + }{ + { + desc: "good cert", + cert: func(*testing.T) tls.Certificate { + return testutil.Certificate(t, testutil.AccAddress(t), com, cod).Cert[0] + }, + }, + { + desc: "empty chain", + cert: func(*testing.T) tls.Certificate { + return tls.Certificate{} + }, + errContains: "empty chain", + }, + { + desc: "invalid subject", + cert: func(t *testing.T) tls.Certificate { + t.Helper() + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: new(big.Int).SetInt64(time.Now().UTC().UnixNano()), + Subject: pkix.Name{ + CommonName: "badcert", + }, + BasicConstraintsValid: true, + } + + certDer, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv) + require.NoError(t, err) + + keyDer, err := x509.MarshalPKCS8PrivateKey(priv) + require.NoError(t, err) + + certBytes := pem.EncodeToMemory(&pem.Block{ + Type: types.PemBlkTypeCertificate, + Bytes: certDer, + }) + privBytes := pem.EncodeToMemory(&pem.Block{ + Type: types.PemBlkTypeECPrivateKey, + Bytes: keyDer, + }) + + cert, err := tls.X509KeyPair(certBytes, privBytes) + require.NoError(t, err) + + return cert + }, + errContains: "invalid certificate's subject", + }, + } + + for _, c := range cases { + c := c + + t.Run(c.desc, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ctx = ContextWithQueryClient(ctx, qclient) + + var m mocks.Client + m.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) + + s := newServer(ctx, crt.Cert, &m) + defer s.Stop() + + l, err := net.Listen("tcp", ":0") + require.NoError(t, err) + + go func() { + require.NoError(t, s.Serve(l)) + }() + + tlsConfig := tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{c.cert(t)}, + } + + conn, err := grpc.DialContext(ctx, l.Addr().String(), + grpc.WithTransportCredentials(credentials.NewTLS(&tlsConfig))) + require.NoError(t, err) + + defer conn.Close() + + _, err = providerv1.NewProviderRPCClient(conn).GetStatus(ctx, &emptypb.Empty{}) + if c.errContains != "" { + assert.ErrorContains(t, err, c.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/gateway/rest/client.go b/gateway/rest/client.go index 44b9f80dc..290e01c22 100644 --- a/gateway/rest/client.go +++ b/gateway/rest/client.go @@ -35,6 +35,7 @@ import ( "github.com/akash-network/provider" cltypes "github.com/akash-network/provider/cluster/types/v1beta3" + "github.com/akash-network/provider/gateway/utils" ) const ( @@ -212,8 +213,10 @@ type ClaimsV1 struct { CertSerialNumber string `json:"cert_serial_number"` } -var errRequiredCertSerialNum = errors.New("cert_serial_number must be present in claims") -var errNonNumericCertSerialNum = errors.New("cert_serial_number must be numeric in claims") +var ( + errRequiredCertSerialNum = errors.New("cert_serial_number must be present in claims") + errNonNumericCertSerialNum = errors.New("cert_serial_number must be numeric in claims") +) func (c *ClientCustomClaims) Valid() error { _, err := sdk.AccAddressFromBech32(c.Subject) @@ -293,63 +296,18 @@ func (c *client) verifyPeerCertificate(certificates [][]byte, _ [][]*x509.Certif return errors.Errorf("tls: invalid certificate chain") } - cert, err := x509.ParseCertificate(certificates[0]) - if err != nil { - return errors.Wrap(err, "tls: failed to parse certificate") - } - - // validation - var prov sdk.Address - if prov, err = sdk.AccAddressFromBech32(cert.Subject.CommonName); err != nil { - return errors.Wrap(err, "tls: invalid certificate's subject common name") - } - - // 1. CommonName in issuer and Subject must be the same - if cert.Subject.CommonName != cert.Issuer.CommonName { - return errors.Wrap(err, "tls: invalid certificate's issuer common name") - } - - if !c.addr.Equals(prov) { - return errors.Errorf("tls: hijacked certificate") - } - - // 2. serial number must be in - if cert.SerialNumber == nil { - return errors.Wrap(err, "tls: invalid certificate serial number") - } - - // 3. look up certificate on chain. it must not be revoked - var resp *ctypes.QueryCertificatesResponse - resp, err = c.cclient.Certificates( + prov, err := utils.VerifyOwnerCertBytes( context.Background(), - &ctypes.QueryCertificatesRequest{ - Filter: ctypes.CertificateFilter{ - Owner: prov.String(), - Serial: cert.SerialNumber.String(), - State: "valid", - }, - }, - ) + certificates, + c.host.Hostname(), + x509.ExtKeyUsageServerAuth, + c.cclient) if err != nil { - return errors.Wrap(err, "tls: unable to fetch certificate from chain") - } - if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { - return errors.New("tls: attempt to use non-existing or revoked certificate") - } - - certPool := x509.NewCertPool() - certPool.AddCert(cert) - - opts := x509.VerifyOptions{ - DNSName: c.host.Hostname(), - Roots: certPool, - CurrentTime: time.Now(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - MaxConstraintComparisions: 0, + return err } - if _, err = cert.Verify(opts); err != nil { - return errors.Wrap(err, "tls: unable to verify certificate") + if !c.addr.Equals(prov) { + return errors.New("tls: hijacked certificate") } return nil @@ -713,8 +671,8 @@ func (c *client) LeaseLogs(ctx context.Context, id mtypes.LeaseID, services string, follow bool, - tailLines int64) (*ServiceLogs, error) { - + tailLines int64, +) (*ServiceLogs, error) { endpoint, err := url.Parse(c.host.String() + "/" + serviceLogsPath(id)) if err != nil { return nil, err @@ -815,3 +773,4 @@ func parseCloseMessage(msg string) string { return "" } + diff --git a/gateway/utils/utils.go b/gateway/utils/utils.go index e030e4c69..05c25c5cc 100644 --- a/gateway/utils/utils.go +++ b/gateway/utils/utils.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "fmt" "time" "github.com/pkg/errors" @@ -22,64 +23,8 @@ func NewServerTLSConfig(ctx context.Context, certs []tls.Certificate, cquery cty InsecureSkipVerify: true, // nolint: gosec MinVersion: tls.VersionTLS13, VerifyPeerCertificate: func(certificates [][]byte, _ [][]*x509.Certificate) error { - if len(certificates) > 0 { - if len(certificates) != 1 { - return errors.Errorf("tls: invalid certificate chain") - } - - cert, err := x509.ParseCertificate(certificates[0]) - if err != nil { - return errors.Wrap(err, "tls: failed to parse certificate") - } - - // validation - var owner sdk.Address - if owner, err = sdk.AccAddressFromBech32(cert.Subject.CommonName); err != nil { - return errors.Wrap(err, "tls: invalid certificate's subject common name") - } - - // 1. CommonName in issuer and Subject must match and be as Bech32 format - if cert.Subject.CommonName != cert.Issuer.CommonName { - return errors.Wrap(err, "tls: invalid certificate's issuer common name") - } - - // 2. serial number must be in - if cert.SerialNumber == nil { - return errors.Wrap(err, "tls: invalid certificate serial number") - } - - // 3. look up certificate on chain - var resp *ctypes.QueryCertificatesResponse - resp, err = cquery.Certificates( - ctx, - &ctypes.QueryCertificatesRequest{ - Filter: ctypes.CertificateFilter{ - Owner: owner.String(), - Serial: cert.SerialNumber.String(), - State: "valid", - }, - }, - ) - if err != nil { - return errors.Wrap(err, "tls: unable to fetch certificate from chain") - } - if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { - return errors.New("tls: attempt to use non-existing or revoked certificate") - } - - clientCertPool := x509.NewCertPool() - clientCertPool.AddCert(cert) - - opts := x509.VerifyOptions{ - Roots: clientCertPool, - CurrentTime: time.Now(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - MaxConstraintComparisions: 0, - } - - if _, err = cert.Verify(opts); err != nil { - return errors.Wrap(err, "tls: unable to verify certificate") - } + if _, err := VerifyOwnerCertBytes(ctx, certificates, "", x509.ExtKeyUsageClientAuth, cquery); err != nil { + return err } return nil }, @@ -87,3 +32,85 @@ func NewServerTLSConfig(ctx context.Context, certs []tls.Certificate, cquery cty return cfg, nil } + +func VerifyOwnerCertBytes(ctx context.Context, chain [][]byte, dnsName string, usage x509.ExtKeyUsage, cquery ctypes.QueryClient) (sdk.Address, error) { + if len(chain) == 0 { + return nil, nil + } + + if len(chain) > 1 { + return nil, errors.Errorf("tls: invalid certificate chain") + } + + c, err := x509.ParseCertificate(chain[0]) + if err != nil { + return nil, fmt.Errorf("tls: failed to parse certificate: %w", err) + } + + return VerifyOwnerCert(ctx, []*x509.Certificate{c}, dnsName, usage, cquery) +} + +func VerifyOwnerCert(ctx context.Context, chain []*x509.Certificate, dnsName string, usage x509.ExtKeyUsage, cquery ctypes.QueryClient) (sdk.Address, error) { + if len(chain) == 0 { + return nil, errors.Errorf("tls: empty chain") + } + + if len(chain) > 1 { + return nil, errors.Errorf("tls: invalid certificate chain") + } + + c := chain[0] + + // validation + owner, err := sdk.AccAddressFromBech32(c.Subject.CommonName) + if err != nil { + return nil, fmt.Errorf("tls: invalid certificate's subject common name: %w", err) + } + + // 1. CommonName in issuer and Subject must match and be as Bech32 format + if c.Subject.CommonName != c.Issuer.CommonName { + return nil, fmt.Errorf("tls: invalid certificate's issuer common name: %w", err) + } + + // 2. serial number must be in + if c.SerialNumber == nil { + return nil, fmt.Errorf("tls: invalid certificate serial number: %w", err) + } + + // 3. look up certificate on chain + var resp *ctypes.QueryCertificatesResponse + resp, err = cquery.Certificates( + ctx, + &ctypes.QueryCertificatesRequest{ + Filter: ctypes.CertificateFilter{ + Owner: owner.String(), + Serial: c.SerialNumber.String(), + State: "valid", + }, + }, + ) + if err != nil { + return nil, fmt.Errorf("tls: unable to fetch certificate from chain: %w", err) + } + if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { + return nil, fmt.Errorf("tls: attempt to use non-existing or revoked certificate: %w", err) + } + + clientCertPool := x509.NewCertPool() + clientCertPool.AddCert(c) + + opts := x509.VerifyOptions{ + DNSName: dnsName, + Roots: clientCertPool, + CurrentTime: time.Now(), + KeyUsages: []x509.ExtKeyUsage{usage}, + MaxConstraintComparisions: 0, + } + + if _, err = c.Verify(opts); err != nil { + return nil, fmt.Errorf("tls: unable to verify certificate: %w", err) + } + + return owner, nil +} + From eeac1af2356847c160fb145b6b3ebb67a2983553 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Tue, 16 Apr 2024 14:11:33 -0600 Subject: [PATCH 02/17] fix: Roll back send-manifest command --- cmd/provider-services/cmd/manifest.go | 83 +++++++-------------------- 1 file changed, 21 insertions(+), 62 deletions(-) diff --git a/cmd/provider-services/cmd/manifest.go b/cmd/provider-services/cmd/manifest.go index 1bac06638..54efa9217 100644 --- a/cmd/provider-services/cmd/manifest.go +++ b/cmd/provider-services/cmd/manifest.go @@ -2,11 +2,9 @@ package cmd import ( "bytes" - "context" "crypto/tls" "encoding/json" "fmt" - "time" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -16,13 +14,10 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" - ptypes "github.com/akash-network/akash-api/go/node/provider/v1beta3" - leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" "github.com/akash-network/node/sdl" cutils "github.com/akash-network/node/x/cert/utils" aclient "github.com/akash-network/provider/client" - gwgrpc "github.com/akash-network/provider/gateway/grpc" gwrest "github.com/akash-network/provider/gateway/rest" ) @@ -97,68 +92,32 @@ func doSendManifest(cmd *cobra.Command, sdlpath string) error { ErrorMessage string `json:"errorMessage,omitempty" yaml:"errorMessage,omitempty"` } - var ( - results = make([]result, len(leases)) - submitFailed = false - ) + results := make([]result, len(leases)) - for i, lid := range leases { - err = func() error { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - provAddr, _ := sdk.AccAddressFromBech32(lid.Provider) - prov, err := cl.Provider(ctx, &ptypes.QueryProviderRequest{Owner: provAddr.String()}) - if err != nil { - return fmt.Errorf("query client provider: %w", err) - } - - hostURIgRPC, err := grpcURI(prov.GetProvider().HostURI) - if err != nil { - return fmt.Errorf("grpc uri: %w", err) - } - - res := result{ - Provider: provAddr, - Status: "PASS", - } - - c, err := gwgrpc.NewClient(ctx, hostURIgRPC, cert, cl) - if err == nil { - defer c.Close() - - if _, err = c.SendManifest(ctx, &leasev1.SendManifestRequest{ - LeaseId: lid, - Manifest: mani, - }); err != nil { - res.Error = err.Error() - res.Status = "FAIL" - submitFailed = true - } - } else { - gclient, err := gwrest.NewClient(cl, provAddr, []tls.Certificate{cert}) - if err != nil { - return fmt.Errorf("gwrest new client: %w", err) - } - - err = gclient.SubmitManifest(cmd.Context(), dseq, mani) - if err != nil { - res.Error = err.Error() - if e, valid := err.(gwrest.ClientResponseError); valid { - res.ErrorMessage = e.Message - } - res.Status = "FAIL" - submitFailed = true - } - } + submitFailed := false - results[i] = res - - return nil - }() + for i, lid := range leases { + prov, _ := sdk.AccAddressFromBech32(lid.Provider) + gclient, err := gwrest.NewClient(cl, prov, []tls.Certificate{cert}) if err != nil { return err } + + err = gclient.SubmitManifest(cmd.Context(), dseq, mani) + res := result{ + Provider: prov, + Status: "PASS", + } + if err != nil { + res.Error = err.Error() + if e, valid := err.(gwrest.ClientResponseError); valid { + res.ErrorMessage = e.Message + } + res.Status = "FAIL" + submitFailed = true + } + + results[i] = res } buf := &bytes.Buffer{} From 61c7af7e1f4c321dfc7d91c825419e9e35e4eb00 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Thu, 25 Apr 2024 12:03:59 -0600 Subject: [PATCH 03/17] fix: Tests --- cmd/provider-services/cmd/leaseLogs.go | 81 ++++++++- cmd/provider-services/cmd/run.go | 7 +- gateway/grpc/client.go | 70 +++++++- gateway/grpc/lease.go | 120 +++++++++++-- gateway/grpc/provider.go | 18 +- gateway/grpc/server.go | 157 ++++++++++++----- gateway/grpc/server_test.go | 231 ++++++++++++++++++------- gateway/rest/client.go | 62 ++++++- gateway/utils/utils.go | 142 +++++++-------- 9 files changed, 653 insertions(+), 235 deletions(-) diff --git a/cmd/provider-services/cmd/leaseLogs.go b/cmd/provider-services/cmd/leaseLogs.go index cecf49249..0f180de46 100644 --- a/cmd/provider-services/cmd/leaseLogs.go +++ b/cmd/provider-services/cmd/leaseLogs.go @@ -1,21 +1,26 @@ package cmd import ( + "context" "crypto/tls" "fmt" "sync" + "time" sdkclient "github.com/cosmos/cosmos-sdk/client" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/akash-network/akash-api/go/node/client/v1beta2" dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" mtypes "github.com/akash-network/akash-api/go/node/market/v1beta4" + ptypes "github.com/akash-network/akash-api/go/node/provider/v1beta3" cmdcommon "github.com/akash-network/node/cmd/common" cutils "github.com/akash-network/node/x/cert/utils" aclient "github.com/akash-network/provider/client" + gwgrpc "github.com/akash-network/provider/gateway/grpc" gwrest "github.com/akash-network/provider/gateway/rest" ) @@ -98,6 +103,72 @@ func doLeaseLogs(cmd *cobra.Command) error { return errors.Errorf("tail flag supplied with invalid value. must be >= -1") } + g := leaseLogGetter{ + cert: cert, + cl: cl, + svcs: svcs, + follow: follow, + tailLines: tailLines, + outputFormat: outputFormat, + cctx: cctx, + } + + if err = g.run(ctx, leases); err != nil { + return fmt.Errorf("getting logs: %w", err) + } + + return nil +} + +type leaseLogGetter struct { + cert tls.Certificate + cl v1beta2.QueryClient + svcs string + follow bool + tailLines int64 + outputFormat string + cctx sdkclient.Context +} + +func (g leaseLogGetter) run(ctx context.Context, leases []mtypes.LeaseID) error { + var ( + restLeases = make([]mtypes.LeaseID, 0, len(leases)) + grpcLeases = make(map[mtypes.LeaseID]*gwgrpc.Client) + ) + + for _, lid := range leases { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + provAddr, _ := sdk.AccAddressFromBech32(lid.Provider) + prov, err := g.cl.Provider(ctx, &ptypes.QueryProviderRequest{Owner: provAddr.String()}) + if err != nil { + return fmt.Errorf("query client provider: %w", err) + } + + hostURIgRPC, err := grpcURI(prov.GetProvider().HostURI) + if err != nil { + return fmt.Errorf("grpc uri: %w", err) + } + + client, err := gwgrpc.NewClient(ctx, hostURIgRPC, g.cert, g.cl) + if err == nil { + grpcLeases[lid] = client + } else { + restLeases = append(restLeases, lid) + } + } + + g.grpc(ctx, grpcLeases) + g.rest(ctx, restLeases) + + return nil +} + +func (g leaseLogGetter) grpc(ctx context.Context, leases map[mtypes.LeaseID]*gwgrpc.Client) { +} + +func (g leaseLogGetter) rest(ctx context.Context, leases []mtypes.LeaseID) { type result struct { lid mtypes.LeaseID error error @@ -109,9 +180,9 @@ func doLeaseLogs(cmd *cobra.Command) error { for _, lid := range leases { stream := result{lid: lid} prov, _ := sdk.AccAddressFromBech32(lid.Provider) - gclient, err := gwrest.NewClient(cl, prov, []tls.Certificate{cert}) + gclient, err := gwrest.NewClient(g.cl, prov, []tls.Certificate{g.cert}) if err == nil { - stream.stream, stream.error = gclient.LeaseLogs(ctx, lid, svcs, follow, tailLines) + stream.stream, stream.error = gclient.LeaseLogs(ctx, lid, g.svcs, g.follow, g.tailLines) } else { stream.error = err } @@ -132,9 +203,9 @@ func doLeaseLogs(cmd *cobra.Command) error { fmt.Printf("[%s][%s] %s\n", evt.Lid, evt.Name, evt.Message) } - if outputFormat == "json" { + if g.outputFormat == "json" { printFn = func(evt logEntry) { - _ = cmdcommon.PrintJSON(cctx, evt) + _ = cmdcommon.PrintJSON(g.cctx, evt) } } @@ -164,6 +235,4 @@ func doLeaseLogs(cmd *cobra.Command) error { wgStreams.Wait() close(outch) - - return nil } diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 0ed899ae0..6a929533e 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -748,7 +748,12 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { ctx = gwgrpc.ContextWithQueryClient(ctx, cl.Query()) - if err = gwgrpc.Serve(ctx, grpcaddr, []tls.Certificate{tlsCert}, service); err != nil { + gs := gwgrpc.NewServer(ctx, + gwgrpc.WithCerts([]tls.Certificate{tlsCert}), + gwgrpc.WithProviderClient(service), + ) + + if err = gs.ServeOn(ctx, grpcaddr); err != nil { return err } diff --git a/gateway/grpc/client.go b/gateway/grpc/client.go index e129f557a..0421bee67 100644 --- a/gateway/grpc/client.go +++ b/gateway/grpc/client.go @@ -4,7 +4,9 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" + "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -12,8 +14,7 @@ import ( ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" providerv1 "github.com/akash-network/akash-api/go/provider/v1" - - "github.com/akash-network/provider/gateway/utils" + sdk "github.com/cosmos/cosmos-sdk/types" ) type Client struct { @@ -31,10 +32,69 @@ func NewClient(ctx context.Context, addr string, cert tls.Certificate, cquery ct tlsConfig := tls.Config{ Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS13, - VerifyPeerCertificate: func(certificates [][]byte, _ [][]*x509.Certificate) error { - if _, err := utils.VerifyOwnerCertBytes(ctx, certificates, "", x509.ExtKeyUsageClientAuth, cquery); err != nil { - return err + VerifyPeerCertificate: func(chain [][]byte, _ [][]*x509.Certificate) error { + if len(chain) == 0 { + return errors.New("tls: empty chain") + } + + if len(chain) > 1 { + return errors.New("tls: invalid certificate chain") + } + + c, err := x509.ParseCertificate(chain[0]) + if err != nil { + return fmt.Errorf("x509 parse certificate: %w", err) + } + + // validation + owner, err := sdk.AccAddressFromBech32(c.Subject.CommonName) + if err != nil { + return fmt.Errorf("tls: invalid certificate's subject common name: %w", err) + } + + // 1. CommonName in issuer and Subject must match and be as Bech32 format + if c.Subject.CommonName != c.Issuer.CommonName { + return fmt.Errorf("tls: invalid certificate's issuer common name: %w", err) + } + + // 2. serial number must be in + if c.SerialNumber == nil { + return fmt.Errorf("tls: invalid certificate serial number: %w", err) + } + + // 3. look up certificate on chain + var resp *ctypes.QueryCertificatesResponse + resp, err = cquery.Certificates( + ctx, + &ctypes.QueryCertificatesRequest{ + Filter: ctypes.CertificateFilter{ + Owner: owner.String(), + Serial: c.SerialNumber.String(), + State: "valid", + }, + }, + ) + if err != nil { + return fmt.Errorf("tls: unable to fetch certificate from chain: %w", err) + } + if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { + return fmt.Errorf("tls: attempt to use non-existing or revoked certificate: %w", err) + } + + clientCertPool := x509.NewCertPool() + clientCertPool.AddCert(c) + + opts := x509.VerifyOptions{ + Roots: clientCertPool, + CurrentTime: time.Now(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + MaxConstraintComparisions: 0, } + + if _, err = c.Verify(opts); err != nil { + return fmt.Errorf("tls: unable to verify certificate: %w", err) + } + return nil }, } diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index fe15446a6..2abfd4f10 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -2,33 +2,131 @@ package grpc import ( "context" + "errors" + "fmt" + "strings" + "sync" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" - "github.com/akash-network/provider" + "github.com/akash-network/provider/cluster/types/v1beta3" ) -type leaseV1 struct { - c provider.Client - ctx context.Context -} +var ErrNoRunningPods = errors.New("no running pods") -func (l *leaseV1) SendManifest(context.Context, *leasev1.SendManifestRequest) (*leasev1.SendManifestResponse, error) { +func (*server) SendManifest(context.Context, *leasev1.SendManifestRequest) (*leasev1.SendManifestResponse, error) { panic("unimplemented") } -func (l *leaseV1) ServiceLogs(context.Context, *leasev1.ServiceLogsRequest) (*leasev1.ServiceLogsResponse, error) { +func (*server) ServiceLogs(context.Context, *leasev1.ServiceLogsRequest) (*leasev1.ServiceLogsResponse, error) { panic("unimplemented") } -func (l *leaseV1) ServiceStatus(context.Context, *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { +func (*server) ServiceStatus(context.Context, *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { panic("unimplemented") } -func (l *leaseV1) StreamServiceLogs(*leasev1.ServiceLogsRequest, leasev1.LeaseRPC_StreamServiceLogsServer) error { - panic("unimplemented") +func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.LeaseRPC_StreamServiceLogsServer) error { + var ( + ctx = strm.Context() + id = r.GetLeaseId() + + // TODO(andrewhare): What is the format of services? + services = strings.Join(r.GetServices(), " ") + + // TODO(andrewhare): Where does lines come from? Is it ignored in follow mode? + lines = int64(111) + ) + + logs, err := s.rc.LeaseLogs(ctx, id, services, true, &lines) + if err != nil { + return fmt.Errorf("lease logs: %w", err) + } + + if len(logs) == 0 { + return ErrNoRunningPods + } + + var ( + errs = make(chan error, len(logs)) + stream = &safeLogStream{s: strm} + ) + + for _, ll := range logs { + go func(l *v1beta3.ServiceLog) { + defer l.Stream.Close() + + for l.Scanner.Scan() { + if ctxErr := ctx.Err(); ctxErr != nil { + if !isContextDoneErr(ctxErr) { + errs <- fmt.Errorf("ctx err: %w", ctxErr) + } + return + } + + var ( + buf = l.Scanner.Bytes() + logs = make([]byte, len(buf)) + ) + + copy(logs, buf) + + resp := leasev1.ServiceLogsResponse{ + Services: []*leasev1.ServiceLogs{ + { + Name: l.Name, + Logs: buf, + }, + }, + } + + if sendErr := stream.Send(&resp); sendErr != nil { + errs <- fmt.Errorf("stream send: %w", sendErr) + return + } + } + + if scanErr := l.Scanner.Err(); scanErr != nil { + errs <- fmt.Errorf("scanner err: %w", err) + return + } + + errs <- nil + }(ll) + } + + var allErr error + for i := 0; i < len(logs); i++ { + allErr = errors.Join(err, <-errs) + } + if allErr != nil { + return fmt.Errorf("scan all logs: %w", err) + } + + return nil } -func (l *leaseV1) StreamServiceStatus(*leasev1.ServiceStatusRequest, leasev1.LeaseRPC_StreamServiceStatusServer) error { +func (*server) StreamServiceStatus(*leasev1.ServiceStatusRequest, leasev1.LeaseRPC_StreamServiceStatusServer) error { panic("unimplemented") } + +type safeLogStream struct { + s leasev1.LeaseRPC_StreamServiceLogsServer + mu sync.Mutex +} + +func (s *safeLogStream) Send(r *leasev1.ServiceLogsResponse) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.s.Send(r); err != nil { + return fmt.Errorf("safe log stream send: %w", err) + } + + return nil +} + +func isContextDoneErr(err error) bool { + return errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) +} diff --git a/gateway/grpc/provider.go b/gateway/grpc/provider.go index 7c6cd1d25..7a15585e0 100644 --- a/gateway/grpc/provider.go +++ b/gateway/grpc/provider.go @@ -5,22 +5,16 @@ import ( "golang.org/x/net/context" "google.golang.org/protobuf/types/known/emptypb" - "github.com/akash-network/provider" "github.com/akash-network/provider/tools/fromctx" ptypes "github.com/akash-network/provider/types" ) -type providerV1 struct { - ctx context.Context - c provider.Client +func (s *server) GetStatus(ctx context.Context, _ *emptypb.Empty) (*providerv1.Status, error) { + return s.pc.StatusV1(ctx) } -func (p *providerV1) GetStatus(ctx context.Context, _ *emptypb.Empty) (*providerv1.Status, error) { - return p.c.StatusV1(ctx) -} - -func (p *providerV1) StreamStatus(_ *emptypb.Empty, stream providerv1.ProviderRPC_StreamStatusServer) error { - bus, err := fromctx.PubSubFromCtx(p.ctx) +func (s *server) StreamStatus(_ *emptypb.Empty, stream providerv1.ProviderRPC_StreamStatusServer) error { + bus, err := fromctx.PubSubFromCtx(s.ctx) if err != nil { return err } @@ -29,8 +23,8 @@ func (p *providerV1) StreamStatus(_ *emptypb.Empty, stream providerv1.ProviderRP for { select { - case <-p.ctx.Done(): - return p.ctx.Err() + case <-s.ctx.Done(): + return s.ctx.Err() case <-stream.Context().Done(): return stream.Context().Err() case evt := <-events: diff --git a/gateway/grpc/server.go b/gateway/grpc/server.go index e62c20553..249290249 100644 --- a/gateway/grpc/server.go +++ b/gateway/grpc/server.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "net" "time" @@ -17,48 +18,60 @@ import ( ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" providerv1 "github.com/akash-network/akash-api/go/provider/v1" - cmblog "github.com/tendermint/tendermint/libs/log" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/akash-network/provider" - "github.com/akash-network/provider/gateway/utils" + "github.com/akash-network/provider/cluster" "github.com/akash-network/provider/tools/fromctx" ) var ( _ providerv1.ProviderRPCServer = (*server)(nil) _ leasev1.LeaseRPCServer = (*server)(nil) + _ Server = (*grpcServer)(nil) ) +type Server interface { + ServeOn(context.Context, string) error +} + +type grpcServer struct { + *grpc.Server +} + type server struct { - *providerV1 - *leaseV1 + ctx context.Context + pc provider.Client + rc cluster.ReadClient + + certs []tls.Certificate + providerClient provider.Client + clusterReadClient cluster.ReadClient } -func Serve(ctx context.Context, endpoint string, certs []tls.Certificate, c provider.Client) error { +func (s *grpcServer) ServeOn(ctx context.Context, addr string) error { group, err := fromctx.ErrGroupFromCtx(ctx) if err != nil { return err } - grpcSrv := newServer(ctx, certs, c) - log := fromctx.LogcFromCtx(ctx) group.Go(func() error { - grpcLis, err := net.Listen("tcp", endpoint) + grpcLis, err := net.Listen("tcp", addr) if err != nil { return err } - log.Info(fmt.Sprintf("grpc listening on \"%s\"", endpoint)) + log.Info(fmt.Sprintf("grpc listening on \"%s\"", addr)) - return grpcSrv.Serve(grpcLis) + return s.Serve(grpcLis) }) group.Go(func() error { <-ctx.Done() - grpcSrv.GracefulStop() + s.GracefulStop() return ctx.Err() }) @@ -66,11 +79,36 @@ func Serve(ctx context.Context, endpoint string, certs []tls.Certificate, c prov return nil } -func newServer(ctx context.Context, certs []tls.Certificate, c provider.Client) *grpc.Server { +type serverOpts struct { + certs []tls.Certificate + providerClient provider.Client + clusterReadClient cluster.ReadClient +} + +type opt func(*serverOpts) + +func WithCerts(c []tls.Certificate) opt { + return func(so *serverOpts) { so.certs = c } +} + +func WithProviderClient(c provider.Client) opt { + return func(so *serverOpts) { so.providerClient = c } +} + +func WithClusterReadClient(c cluster.ReadClient) opt { + return func(so *serverOpts) { so.clusterReadClient = c } +} + +func NewServer(ctx context.Context, opts ...opt) Server { + var o serverOpts + for _, opt := range opts { + opt(&o) + } + // InsecureSkipVerify is set to true due to inability to use normal TLS verification // certificate validation and authentication performed later in mtlsHandler tlsConfig := &tls.Config{ - Certificates: certs, + Certificates: o.certs, ClientAuth: tls.RequestClientCert, InsecureSkipVerify: true, // nolint: gosec MinVersion: tls.VersionTLS13, @@ -88,58 +126,91 @@ func newServer(ctx context.Context, certs []tls.Certificate, c provider.Client) }), grpc.ChainUnaryInterceptor( mtlsInterceptor(cquery), - errorLogInterceptor(fromctx.LogcFromCtx(ctx)), ), ) s := &server{ - providerV1: &providerV1{ - ctx: ctx, - c: c, - }, - leaseV1: &leaseV1{ - ctx: ctx, - c: c, - }, + ctx: ctx, + pc: o.providerClient, + rc: o.clusterReadClient, } providerv1.RegisterProviderRPCServer(g, s) leasev1.RegisterLeaseRPCServer(g, s) gogoreflection.Register(g) - return g + return &grpcServer{Server: g} } func mtlsInterceptor(cquery ctypes.QueryClient) grpc.UnaryServerInterceptor { - return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { + return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { if p, ok := peer.FromContext(ctx); ok { if mtls, ok := p.AuthInfo.(credentials.TLSInfo); ok { - owner, err := utils.VerifyOwnerCert(ctx, mtls.State.PeerCertificates, "", x509.ExtKeyUsageClientAuth, cquery) + certificates := mtls.State.PeerCertificates + + if len(certificates) == 0 { + return nil, errors.New("tls: empty chain") + } + + if len(certificates) != 1 { + return nil, errors.New("tls: invalid certificate chain") // nolint: goerr113 + } + + cert := certificates[0] + + // validation + var owner sdk.Address + if owner, err = sdk.AccAddressFromBech32(cert.Subject.CommonName); err != nil { + return nil, fmt.Errorf("tls: invalid certificate's subject common name: %w", err) + } + + // 1. CommonName in issuer and Subject must match and be as Bech32 format + if cert.Subject.CommonName != cert.Issuer.CommonName { + return nil, fmt.Errorf("tls: invalid certificate's issuer common name: %w", err) + } + + // 2. serial number must be in + if cert.SerialNumber == nil { + return nil, fmt.Errorf("tls: invalid certificate serial number: %w", err) + } + + // 3. look up certificate on chain + var resp *ctypes.QueryCertificatesResponse + resp, err = cquery.Certificates( + ctx, + &ctypes.QueryCertificatesRequest{ + Filter: ctypes.CertificateFilter{ + Owner: owner.String(), + Serial: cert.SerialNumber.String(), + State: "valid", + }, + }, + ) if err != nil { - return nil, fmt.Errorf("verify cert chain: %w", err) + return nil, fmt.Errorf("tls: unable to fetch certificate from chain: %w", err) + } + if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { + return nil, errors.New("tls: attempt to use non-existing or revoked certificate") // nolint: goerr113 } - if owner != nil { - ctx = ContextWithOwner(ctx, owner) + clientCertPool := x509.NewCertPool() + clientCertPool.AddCert(cert) + + opts := x509.VerifyOptions{ + Roots: clientCertPool, + CurrentTime: time.Now(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + MaxConstraintComparisions: 0, } - } - } - return next(ctx, req) - } -} + if _, err = cert.Verify(opts); err != nil { + return nil, fmt.Errorf("tls: unable to verify certificate: %w", err) + } -// TODO(andrewhare): Possibly replace this with -// https://github.com/grpc-ecosystem/go-grpc-middleware/tree/main/interceptors/logging -// to get full request/response logging? -func errorLogInterceptor(l cmblog.Logger) grpc.UnaryServerInterceptor { - return func(ctx context.Context, req any, i *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) { - resp, err := next(ctx, req) - if err != nil { - l.Error(i.FullMethod, "err", err) + ctx = ContextWithOwner(ctx, owner) + } } - return resp, err + return handler(ctx, req) } } - diff --git a/gateway/grpc/server_test.go b/gateway/grpc/server_test.go index 6780171cf..ea35f542d 100644 --- a/gateway/grpc/server_test.go +++ b/gateway/grpc/server_test.go @@ -1,18 +1,14 @@ package grpc import ( + "bufio" "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "math/big" + "errors" + "io" "net" + "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -27,13 +23,12 @@ import ( providerv1 "github.com/akash-network/akash-api/go/provider/v1" "github.com/akash-network/node/testutil" + cmocks "github.com/akash-network/provider/cluster/mocks" + "github.com/akash-network/provider/cluster/types/v1beta3" "github.com/akash-network/provider/mocks" + pmocks "github.com/akash-network/provider/mocks" ) -type asserter interface { - AssertExpectations(mock.TestingT) bool -} - type client struct { p providerv1.ProviderRPCClient l leasev1.LeaseRPCClient @@ -65,22 +60,152 @@ func TestRPCs(t *testing.T) { }, nil) cases := []struct { - desc string - mocks func() (*mocks.Client, []asserter) - run func(context.Context, *testing.T, client) + desc string + providerClient func() *pmocks.Client + readClient func() *cmocks.ReadClient + run func(context.Context, *testing.T, client) }{ { desc: "GetStatus", - mocks: func() (*mocks.Client, []asserter) { - var c mocks.Client - c.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) - return &c, nil + providerClient: func() *mocks.Client { + var m mocks.Client + m.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) + return &m }, run: func(ctx context.Context, t *testing.T, c client) { _, err := c.p.GetStatus(ctx, &emptypb.Empty{}) assert.NoError(t, err) }, }, + { + desc: "StreamServiceLogs none", + readClient: func() *cmocks.ReadClient { + var m cmocks.ReadClient + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*v1beta3.ServiceLog{}, nil) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + s, err := c.l.StreamServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + require.NoError(t, err) + + _, err = s.Recv() + assert.ErrorContains(t, err, ErrNoRunningPods.Error()) + }, + }, + { + desc: "StreamServiceLogs LeaseLogs err", + readClient: func() *cmocks.ReadClient { + var m cmocks.ReadClient + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, errors.New("boom")) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + s, err := c.l.StreamServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + require.NoError(t, err) + + _, err = s.Recv() + assert.ErrorContains(t, err, "boom") + }, + }, + { + desc: "StreamServiceLogs one", + readClient: func() *cmocks.ReadClient { + var m cmocks.ReadClient + + stream := io.NopCloser(strings.NewReader("1\n2\n3")) + + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*v1beta3.ServiceLog{ + { + Name: "one", + Stream: stream, + Scanner: bufio.NewScanner(stream), + }, + }, nil) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + s, err := c.l.StreamServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + require.NoError(t, err) + + var ( + expected = []string{"1", "2", "3"} + actual = make([]string, 0, len(expected)) + ) + + for { + r, err := s.Recv() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + actual = append(actual, string(r.Services[0].Logs)) + } + + assert.Equal(t, expected, actual) + }, + }, + { + desc: "StreamServiceLogs multiple", + readClient: func() *cmocks.ReadClient { + var m cmocks.ReadClient + + var ( + stream1 = io.NopCloser(strings.NewReader("1_1\n1_2\n1_3")) + stream2 = io.NopCloser(strings.NewReader("2_1\n2_2\n2_3")) + stream3 = io.NopCloser(strings.NewReader("3_1\n3_2\n3_3")) + ) + + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*v1beta3.ServiceLog{ + { + Name: "one", + Stream: stream1, + Scanner: bufio.NewScanner(stream1), + }, + { + Name: "two", + Stream: stream2, + Scanner: bufio.NewScanner(stream2), + }, + { + Name: "three", + Stream: stream3, + Scanner: bufio.NewScanner(stream3), + }, + }, nil) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + s, err := c.l.StreamServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + require.NoError(t, err) + + var ( + expected = map[string][]string{ + "one": {"1_1", "1_2", "1_3"}, + "two": {"2_1", "2_2", "2_3"}, + "three": {"3_1", "3_2", "3_3"}, + } + actual = make(map[string][]string) + ) + + for { + r, err := s.Recv() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + + for _, s := range r.Services { + actual[s.Name] = append(actual[s.Name], string(s.Logs)) + } + } + + assert.Equal(t, expected, actual) + }, + }, } for _, c := range cases { @@ -92,14 +217,26 @@ func TestRPCs(t *testing.T) { ctx = ContextWithQueryClient(ctx, qclient) - mc, as := c.mocks() - defer mc.AssertExpectations(t) + var ( + pc *pmocks.Client + rc *cmocks.ReadClient + ) - for _, a := range as { - defer a.AssertExpectations(t) + if c.providerClient != nil { + pc = c.providerClient() + defer pc.AssertExpectations(t) + } + if c.readClient != nil { + rc = c.readClient() + defer rc.AssertExpectations(t) } - s := newServer(ctx, crt1.Cert, mc) + s := NewServer(ctx, + WithCerts(crt1.Cert), + WithProviderClient(pc), + WithClusterReadClient(rc), + ).(*grpcServer) + defer s.Stop() l, err := net.Listen("tcp", ":0") @@ -168,44 +305,6 @@ func TestMTLS(t *testing.T) { }, errContains: "empty chain", }, - { - desc: "invalid subject", - cert: func(t *testing.T) tls.Certificate { - t.Helper() - - priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - require.NoError(t, err) - - template := x509.Certificate{ - SerialNumber: new(big.Int).SetInt64(time.Now().UTC().UnixNano()), - Subject: pkix.Name{ - CommonName: "badcert", - }, - BasicConstraintsValid: true, - } - - certDer, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv) - require.NoError(t, err) - - keyDer, err := x509.MarshalPKCS8PrivateKey(priv) - require.NoError(t, err) - - certBytes := pem.EncodeToMemory(&pem.Block{ - Type: types.PemBlkTypeCertificate, - Bytes: certDer, - }) - privBytes := pem.EncodeToMemory(&pem.Block{ - Type: types.PemBlkTypeECPrivateKey, - Bytes: keyDer, - }) - - cert, err := tls.X509KeyPair(certBytes, privBytes) - require.NoError(t, err) - - return cert - }, - errContains: "invalid certificate's subject", - }, } for _, c := range cases { @@ -217,10 +316,14 @@ func TestMTLS(t *testing.T) { ctx = ContextWithQueryClient(ctx, qclient) - var m mocks.Client + var m pmocks.Client m.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) - s := newServer(ctx, crt.Cert, &m) + s := NewServer(ctx, + WithCerts(crt.Cert), + WithProviderClient(&m), + ).(*grpcServer) + defer s.Stop() l, err := net.Listen("tcp", ":0") diff --git a/gateway/rest/client.go b/gateway/rest/client.go index 290e01c22..7c7441246 100644 --- a/gateway/rest/client.go +++ b/gateway/rest/client.go @@ -35,7 +35,6 @@ import ( "github.com/akash-network/provider" cltypes "github.com/akash-network/provider/cluster/types/v1beta3" - "github.com/akash-network/provider/gateway/utils" ) const ( @@ -296,18 +295,63 @@ func (c *client) verifyPeerCertificate(certificates [][]byte, _ [][]*x509.Certif return errors.Errorf("tls: invalid certificate chain") } - prov, err := utils.VerifyOwnerCertBytes( - context.Background(), - certificates, - c.host.Hostname(), - x509.ExtKeyUsageServerAuth, - c.cclient) + cert, err := x509.ParseCertificate(certificates[0]) if err != nil { - return err + return errors.Wrap(err, "tls: failed to parse certificate") + } + + // validation + var prov sdk.Address + if prov, err = sdk.AccAddressFromBech32(cert.Subject.CommonName); err != nil { + return errors.Wrap(err, "tls: invalid certificate's subject common name") + } + + // 1. CommonName in issuer and Subject must be the same + if cert.Subject.CommonName != cert.Issuer.CommonName { + return errors.Wrap(err, "tls: invalid certificate's issuer common name") } if !c.addr.Equals(prov) { - return errors.New("tls: hijacked certificate") + return errors.Errorf("tls: hijacked certificate") + } + + // 2. serial number must be in + if cert.SerialNumber == nil { + return errors.Wrap(err, "tls: invalid certificate serial number") + } + + // 3. look up certificate on chain. it must not be revoked + var resp *ctypes.QueryCertificatesResponse + resp, err = c.cclient.Certificates( + context.Background(), + &ctypes.QueryCertificatesRequest{ + Filter: ctypes.CertificateFilter{ + Owner: prov.String(), + Serial: cert.SerialNumber.String(), + State: "valid", + }, + }, + ) + if err != nil { + return errors.Wrap(err, "tls: unable to fetch certificate from chain") + } + if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { + return errors.New("tls: attempt to use non-existing or revoked certificate") + } + + certPool := x509.NewCertPool() + certPool.AddCert(cert) + + opts := x509.VerifyOptions{ + DNSName: c.host.Hostname(), + Roots: certPool, + CurrentTime: time.Now(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + MaxConstraintComparisions: 0, + } + + if _, err = cert.Verify(opts); err != nil { + return errors.Wrap(err, "tls: unable to verify certificate") } return nil diff --git a/gateway/utils/utils.go b/gateway/utils/utils.go index 05c25c5cc..2b3ccfb9b 100644 --- a/gateway/utils/utils.go +++ b/gateway/utils/utils.go @@ -4,7 +4,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "fmt" "time" "github.com/pkg/errors" @@ -23,8 +22,64 @@ func NewServerTLSConfig(ctx context.Context, certs []tls.Certificate, cquery cty InsecureSkipVerify: true, // nolint: gosec MinVersion: tls.VersionTLS13, VerifyPeerCertificate: func(certificates [][]byte, _ [][]*x509.Certificate) error { - if _, err := VerifyOwnerCertBytes(ctx, certificates, "", x509.ExtKeyUsageClientAuth, cquery); err != nil { - return err + if len(certificates) > 0 { + if len(certificates) != 1 { + return errors.Errorf("tls: invalid certificate chain") + } + + cert, err := x509.ParseCertificate(certificates[0]) + if err != nil { + return errors.Wrap(err, "tls: failed to parse certificate") + } + + // validation + var owner sdk.Address + if owner, err = sdk.AccAddressFromBech32(cert.Subject.CommonName); err != nil { + return errors.Wrap(err, "tls: invalid certificate's subject common name") + } + + // 1. CommonName in issuer and Subject must match and be as Bech32 format + if cert.Subject.CommonName != cert.Issuer.CommonName { + return errors.Wrap(err, "tls: invalid certificate's issuer common name") + } + + // 2. serial number must be in + if cert.SerialNumber == nil { + return errors.Wrap(err, "tls: invalid certificate serial number") + } + + // 3. look up certificate on chain + var resp *ctypes.QueryCertificatesResponse + resp, err = cquery.Certificates( + ctx, + &ctypes.QueryCertificatesRequest{ + Filter: ctypes.CertificateFilter{ + Owner: owner.String(), + Serial: cert.SerialNumber.String(), + State: "valid", + }, + }, + ) + if err != nil { + return errors.Wrap(err, "tls: unable to fetch certificate from chain") + } + if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { + return errors.New("tls: attempt to use non-existing or revoked certificate") + } + + clientCertPool := x509.NewCertPool() + clientCertPool.AddCert(cert) + + opts := x509.VerifyOptions{ + Roots: clientCertPool, + CurrentTime: time.Now(), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + MaxConstraintComparisions: 0, + } + + if _, err = cert.Verify(opts); err != nil { + return errors.Wrap(err, "tls: unable to verify certificate") + } } return nil }, @@ -33,84 +88,3 @@ func NewServerTLSConfig(ctx context.Context, certs []tls.Certificate, cquery cty return cfg, nil } -func VerifyOwnerCertBytes(ctx context.Context, chain [][]byte, dnsName string, usage x509.ExtKeyUsage, cquery ctypes.QueryClient) (sdk.Address, error) { - if len(chain) == 0 { - return nil, nil - } - - if len(chain) > 1 { - return nil, errors.Errorf("tls: invalid certificate chain") - } - - c, err := x509.ParseCertificate(chain[0]) - if err != nil { - return nil, fmt.Errorf("tls: failed to parse certificate: %w", err) - } - - return VerifyOwnerCert(ctx, []*x509.Certificate{c}, dnsName, usage, cquery) -} - -func VerifyOwnerCert(ctx context.Context, chain []*x509.Certificate, dnsName string, usage x509.ExtKeyUsage, cquery ctypes.QueryClient) (sdk.Address, error) { - if len(chain) == 0 { - return nil, errors.Errorf("tls: empty chain") - } - - if len(chain) > 1 { - return nil, errors.Errorf("tls: invalid certificate chain") - } - - c := chain[0] - - // validation - owner, err := sdk.AccAddressFromBech32(c.Subject.CommonName) - if err != nil { - return nil, fmt.Errorf("tls: invalid certificate's subject common name: %w", err) - } - - // 1. CommonName in issuer and Subject must match and be as Bech32 format - if c.Subject.CommonName != c.Issuer.CommonName { - return nil, fmt.Errorf("tls: invalid certificate's issuer common name: %w", err) - } - - // 2. serial number must be in - if c.SerialNumber == nil { - return nil, fmt.Errorf("tls: invalid certificate serial number: %w", err) - } - - // 3. look up certificate on chain - var resp *ctypes.QueryCertificatesResponse - resp, err = cquery.Certificates( - ctx, - &ctypes.QueryCertificatesRequest{ - Filter: ctypes.CertificateFilter{ - Owner: owner.String(), - Serial: c.SerialNumber.String(), - State: "valid", - }, - }, - ) - if err != nil { - return nil, fmt.Errorf("tls: unable to fetch certificate from chain: %w", err) - } - if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { - return nil, fmt.Errorf("tls: attempt to use non-existing or revoked certificate: %w", err) - } - - clientCertPool := x509.NewCertPool() - clientCertPool.AddCert(c) - - opts := x509.VerifyOptions{ - DNSName: dnsName, - Roots: clientCertPool, - CurrentTime: time.Now(), - KeyUsages: []x509.ExtKeyUsage{usage}, - MaxConstraintComparisions: 0, - } - - if _, err = c.Verify(opts); err != nil { - return nil, fmt.Errorf("tls: unable to verify certificate: %w", err) - } - - return owner, nil -} - From 1b23eb6b0fb341df5fa9e91e2d96fd50794261b3 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Thu, 25 Apr 2024 12:09:10 -0600 Subject: [PATCH 04/17] fix: Reset formatting on files --- gateway/rest/client.go | 11 ++++------- gateway/utils/utils.go | 1 - 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/gateway/rest/client.go b/gateway/rest/client.go index 7c7441246..44b9f80dc 100644 --- a/gateway/rest/client.go +++ b/gateway/rest/client.go @@ -212,10 +212,8 @@ type ClaimsV1 struct { CertSerialNumber string `json:"cert_serial_number"` } -var ( - errRequiredCertSerialNum = errors.New("cert_serial_number must be present in claims") - errNonNumericCertSerialNum = errors.New("cert_serial_number must be numeric in claims") -) +var errRequiredCertSerialNum = errors.New("cert_serial_number must be present in claims") +var errNonNumericCertSerialNum = errors.New("cert_serial_number must be numeric in claims") func (c *ClientCustomClaims) Valid() error { _, err := sdk.AccAddressFromBech32(c.Subject) @@ -715,8 +713,8 @@ func (c *client) LeaseLogs(ctx context.Context, id mtypes.LeaseID, services string, follow bool, - tailLines int64, -) (*ServiceLogs, error) { + tailLines int64) (*ServiceLogs, error) { + endpoint, err := url.Parse(c.host.String() + "/" + serviceLogsPath(id)) if err != nil { return nil, err @@ -817,4 +815,3 @@ func parseCloseMessage(msg string) string { return "" } - diff --git a/gateway/utils/utils.go b/gateway/utils/utils.go index 2b3ccfb9b..e030e4c69 100644 --- a/gateway/utils/utils.go +++ b/gateway/utils/utils.go @@ -87,4 +87,3 @@ func NewServerTLSConfig(ctx context.Context, certs []tls.Certificate, cquery cty return cfg, nil } - From 84461842de618f1145acdc7d971d9f4020db7ef5 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Thu, 25 Apr 2024 15:50:46 -0600 Subject: [PATCH 05/17] feat: Wire up client --- cmd/provider-services/cmd/leaseLogs.go | 125 ++++++++++++++++++------- gateway/grpc/lease.go | 8 +- 2 files changed, 92 insertions(+), 41 deletions(-) diff --git a/cmd/provider-services/cmd/leaseLogs.go b/cmd/provider-services/cmd/leaseLogs.go index 0f180de46..59076a6cb 100644 --- a/cmd/provider-services/cmd/leaseLogs.go +++ b/cmd/provider-services/cmd/leaseLogs.go @@ -3,19 +3,22 @@ package cmd import ( "context" "crypto/tls" + "errors" "fmt" + "io" + "strings" "sync" "time" sdkclient "github.com/cosmos/cosmos-sdk/client" sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/akash-network/akash-api/go/node/client/v1beta2" dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" mtypes "github.com/akash-network/akash-api/go/node/market/v1beta4" ptypes "github.com/akash-network/akash-api/go/node/provider/v1beta3" + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" cmdcommon "github.com/akash-network/node/cmd/common" cutils "github.com/akash-network/node/x/cert/utils" @@ -86,7 +89,7 @@ func doLeaseLogs(cmd *cobra.Command) error { } if outputFormat != outputText && outputFormat != outputJSON { - return errors.Errorf("invalid output format %s. expected text|json", outputFormat) + return fmt.Errorf("invalid output format %s. expected text|json", outputFormat) } follow, err := cmd.Flags().GetBool(flagFollow) @@ -100,17 +103,19 @@ func doLeaseLogs(cmd *cobra.Command) error { } if tailLines < -1 { - return errors.Errorf("tail flag supplied with invalid value. must be >= -1") + return fmt.Errorf("tail flag supplied with invalid value. must be >= -1") } g := leaseLogGetter{ - cert: cert, - cl: cl, - svcs: svcs, - follow: follow, - tailLines: tailLines, - outputFormat: outputFormat, - cctx: cctx, + cert: cert, + cl: cl, + svcs: svcs, + follow: follow, + tailLines: tailLines, + printer: printer{ + cctx: cctx, + fmt: outputFormat, + }, } if err = g.run(ctx, leases); err != nil { @@ -121,19 +126,18 @@ func doLeaseLogs(cmd *cobra.Command) error { } type leaseLogGetter struct { - cert tls.Certificate - cl v1beta2.QueryClient - svcs string - follow bool - tailLines int64 - outputFormat string - cctx sdkclient.Context + cert tls.Certificate + cl v1beta2.QueryClient + svcs string + follow bool + tailLines int64 + printer printer } func (g leaseLogGetter) run(ctx context.Context, leases []mtypes.LeaseID) error { var ( restLeases = make([]mtypes.LeaseID, 0, len(leases)) - grpcLeases = make(map[mtypes.LeaseID]*gwgrpc.Client) + grpcLeases = make(map[mtypes.LeaseID]*gwgrpc.Client, len(leases)) ) for _, lid := range leases { @@ -166,6 +170,50 @@ func (g leaseLogGetter) run(ctx context.Context, leases []mtypes.LeaseID) error } func (g leaseLogGetter) grpc(ctx context.Context, leases map[mtypes.LeaseID]*gwgrpc.Client) { + var wg sync.WaitGroup + wg.Add(len(leases)) + + for lid, cc := range leases { + go func(c *gwgrpc.Client, id mtypes.LeaseID) { + defer wg.Done() + + req := leasev1.ServiceLogsRequest{ + Services: strings.Split(g.svcs, " "), + LeaseId: id, + } + + logErr := func(err error) { + fmt.Printf("lease %v: %v", id, err) + } + + s, err := c.StreamServiceLogs(ctx, &req) + if err != nil { + logErr(fmt.Errorf("stream service logs: %w", err)) + return + } + + for { + r, err := s.Recv() + switch { + case errors.Is(err, io.EOF): + break + case err != nil: + logErr(fmt.Errorf("recv: %w", err)) + return + } + + for _, s := range r.Services { + g.printer.write(logEntry{ + Name: s.GetName(), + Message: string(s.GetLogs()), + Lid: id, + }) + } + } + }(cc, lid) + } + + wg.Wait() } func (g leaseLogGetter) rest(ctx context.Context, leases []mtypes.LeaseID) { @@ -192,26 +240,11 @@ func (g leaseLogGetter) rest(ctx context.Context, leases []mtypes.LeaseID) { var wgStreams sync.WaitGroup - type logEntry struct { - gwrest.ServiceLogMessage `json:",inline"` - Lid mtypes.LeaseID `json:"lease_id"` - } - outch := make(chan logEntry) - printFn := func(evt logEntry) { - fmt.Printf("[%s][%s] %s\n", evt.Lid, evt.Name, evt.Message) - } - - if g.outputFormat == "json" { - printFn = func(evt logEntry) { - _ = cmdcommon.PrintJSON(g.cctx, evt) - } - } - go func() { for evt := range outch { - printFn(evt) + g.printer.write(evt) } }() @@ -226,8 +259,9 @@ func (g leaseLogGetter) rest(ctx context.Context, leases []mtypes.LeaseID) { for res := range stream.stream.Stream { outch <- logEntry{ - ServiceLogMessage: res, - Lid: stream.lid, + Name: res.Name, + Message: res.Message, + Lid: stream.lid, } } }(stream) @@ -236,3 +270,22 @@ func (g leaseLogGetter) rest(ctx context.Context, leases []mtypes.LeaseID) { wgStreams.Wait() close(outch) } + +type logEntry struct { + Name string `json:"name"` + Message string `json:"message"` + Lid mtypes.LeaseID `json:"lease_id"` +} + +type printer struct { + fmt string + cctx sdkclient.Context +} + +func (p printer) write(e logEntry) { + if p.fmt == "json" { + cmdcommon.PrintJSON(p.cctx, e) + } else { + fmt.Printf("[%s][%s] %s\n", e.Lid, e.Name, e.Message) + } +} diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 2abfd4f10..2e9a2a407 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -28,13 +28,11 @@ func (*server) ServiceStatus(context.Context, *leasev1.ServiceStatusRequest) (*l func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.LeaseRPC_StreamServiceLogsServer) error { var ( - ctx = strm.Context() - id = r.GetLeaseId() - - // TODO(andrewhare): What is the format of services? + ctx = strm.Context() + id = r.GetLeaseId() services = strings.Join(r.GetServices(), " ") - // TODO(andrewhare): Where does lines come from? Is it ignored in follow mode? + // TODO(andrewhare): Where does this value come from? Is it ignored in follow mode? lines = int64(111) ) From e33d605bced5bc7a742a8898bd2209dd2ef4c2b0 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Fri, 26 Apr 2024 15:04:19 -0600 Subject: [PATCH 06/17] fix: Bugs --- cmd/provider-services/cmd/grpc.go | 4 +- cmd/provider-services/cmd/leaseLogs.go | 49 ++++++++++++---------- cmd/provider-services/cmd/run.go | 1 + gateway/grpc/client.go | 5 ++- gateway/grpc/lease.go | 58 +++++++++++--------------- 5 files changed, 60 insertions(+), 57 deletions(-) diff --git a/cmd/provider-services/cmd/grpc.go b/cmd/provider-services/cmd/grpc.go index f88cc8e28..146e2a2ad 100644 --- a/cmd/provider-services/cmd/grpc.go +++ b/cmd/provider-services/cmd/grpc.go @@ -6,6 +6,8 @@ import ( "net/url" ) +const gRPCDefaultPort = "8444" + func grpcURI(hostURI string) (string, error) { u, err := url.Parse(hostURI) if err != nil { @@ -17,5 +19,5 @@ func grpcURI(hostURI string) (string, error) { return "", fmt.Errorf("split host port: %w", err) } - return net.JoinHostPort(h, "8444"), nil + return net.JoinHostPort(h, gRPCDefaultPort), nil } diff --git a/cmd/provider-services/cmd/leaseLogs.go b/cmd/provider-services/cmd/leaseLogs.go index 59076a6cb..fbed03507 100644 --- a/cmd/provider-services/cmd/leaseLogs.go +++ b/cmd/provider-services/cmd/leaseLogs.go @@ -3,9 +3,7 @@ package cmd import ( "context" "crypto/tls" - "errors" "fmt" - "io" "strings" "sync" "time" @@ -13,6 +11,8 @@ import ( sdkclient "github.com/cosmos/cosmos-sdk/client" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/akash-network/akash-api/go/node/client/v1beta2" dtypes "github.com/akash-network/akash-api/go/node/deployment/v1beta3" @@ -141,9 +141,6 @@ func (g leaseLogGetter) run(ctx context.Context, leases []mtypes.LeaseID) error ) for _, lid := range leases { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - provAddr, _ := sdk.AccAddressFromBech32(lid.Provider) prov, err := g.cl.Provider(ctx, &ptypes.QueryProviderRequest{Owner: provAddr.String()}) if err != nil { @@ -155,7 +152,10 @@ func (g leaseLogGetter) run(ctx context.Context, leases []mtypes.LeaseID) error return fmt.Errorf("grpc uri: %w", err) } - client, err := gwgrpc.NewClient(ctx, hostURIgRPC, g.cert, g.cl) + ctxDial, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + client, err := gwgrpc.NewClient(ctxDial, hostURIgRPC, g.cert, g.cl) if err == nil { grpcLeases[lid] = client } else { @@ -183,7 +183,7 @@ func (g leaseLogGetter) grpc(ctx context.Context, leases map[mtypes.LeaseID]*gwg } logErr := func(err error) { - fmt.Printf("lease %v: %v", id, err) + fmt.Printf("[%s]: %v", id, err) } s, err := c.StreamServiceLogs(ctx, &req) @@ -193,21 +193,28 @@ func (g leaseLogGetter) grpc(ctx context.Context, leases map[mtypes.LeaseID]*gwg } for { - r, err := s.Recv() - switch { - case errors.Is(err, io.EOF): - break - case err != nil: - logErr(fmt.Errorf("recv: %w", err)) + select { + case <-ctx.Done(): return - } - - for _, s := range r.Services { - g.printer.write(logEntry{ - Name: s.GetName(), - Message: string(s.GetLogs()), - Lid: id, - }) + default: + r, err := s.Recv() + if err != nil { + if e, ok := status.FromError(err); ok { + if e.Code() != codes.Canceled { + logErr(fmt.Errorf("recv: %w", err)) + } + } + + return + } + + for _, s := range r.Services { + g.printer.write(logEntry{ + Name: s.GetName(), + Message: string(s.GetLogs()), + Lid: id, + }) + } } } }(cc, lid) diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 6a929533e..523c5ee68 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -751,6 +751,7 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { gs := gwgrpc.NewServer(ctx, gwgrpc.WithCerts([]tls.Certificate{tlsCert}), gwgrpc.WithProviderClient(service), + gwgrpc.WithClusterReadClient(service.Cluster()), ) if err = gs.ServeOn(ctx, grpcaddr); err != nil { diff --git a/gateway/grpc/client.go b/gateway/grpc/client.go index 0421bee67..4639d3419 100644 --- a/gateway/grpc/client.go +++ b/gateway/grpc/client.go @@ -30,8 +30,9 @@ func (c *Client) Close() error { func NewClient(ctx context.Context, addr string, cert tls.Certificate, cquery ctypes.QueryClient) (*Client, error) { tlsConfig := tls.Config{ - Certificates: []tls.Certificate{cert}, - MinVersion: tls.VersionTLS13, + InsecureSkipVerify: true, + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS13, VerifyPeerCertificate: func(chain [][]byte, _ [][]*x509.Certificate) error { if len(chain) == 0 { return errors.New("tls: empty chain") diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 2e9a2a407..965c17a94 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -53,43 +53,40 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L for _, ll := range logs { go func(l *v1beta3.ServiceLog) { defer l.Stream.Close() + defer func() { errs <- nil }() for l.Scanner.Scan() { - if ctxErr := ctx.Err(); ctxErr != nil { - if !isContextDoneErr(ctxErr) { - errs <- fmt.Errorf("ctx err: %w", ctxErr) - } + select { + case <-ctx.Done(): return - } - - var ( - buf = l.Scanner.Bytes() - logs = make([]byte, len(buf)) - ) - - copy(logs, buf) - - resp := leasev1.ServiceLogsResponse{ - Services: []*leasev1.ServiceLogs{ - { - Name: l.Name, - Logs: buf, + default: + var ( + buf = l.Scanner.Bytes() + logs = make([]byte, len(buf)) + ) + + copy(logs, buf) + + resp := leasev1.ServiceLogsResponse{ + Services: []*leasev1.ServiceLogs{ + { + Name: l.Name, + Logs: buf, + }, }, - }, + } + + if sendErr := stream.Send(&resp); sendErr != nil { + errs <- fmt.Errorf("stream send: %w", sendErr) + return + } } - if sendErr := stream.Send(&resp); sendErr != nil { - errs <- fmt.Errorf("stream send: %w", sendErr) + if scanErr := l.Scanner.Err(); scanErr != nil { + errs <- fmt.Errorf("scanner err: %w", err) return } } - - if scanErr := l.Scanner.Err(); scanErr != nil { - errs <- fmt.Errorf("scanner err: %w", err) - return - } - - errs <- nil }(ll) } @@ -123,8 +120,3 @@ func (s *safeLogStream) Send(r *leasev1.ServiceLogsResponse) error { return nil } - -func isContextDoneErr(err error) bool { - return errors.Is(err, context.Canceled) || - errors.Is(err, context.DeadlineExceeded) -} From aa72e2dbc57510cd2870f774028f074ceb29ead9 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Fri, 26 Apr 2024 15:07:10 -0600 Subject: [PATCH 07/17] fix: Set to 0 --- gateway/grpc/lease.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 965c17a94..fcb300c2e 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -33,7 +33,7 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L services = strings.Join(r.GetServices(), " ") // TODO(andrewhare): Where does this value come from? Is it ignored in follow mode? - lines = int64(111) + lines = int64(0) ) logs, err := s.rc.LeaseLogs(ctx, id, services, true, &lines) From ef24fc223def24047297507ce9008b25ff015a8f Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Fri, 26 Apr 2024 15:25:25 -0600 Subject: [PATCH 08/17] fix: Remove TODO --- gateway/grpc/lease.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index fcb300c2e..d4105eb1c 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -31,9 +31,7 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L ctx = strm.Context() id = r.GetLeaseId() services = strings.Join(r.GetServices(), " ") - - // TODO(andrewhare): Where does this value come from? Is it ignored in follow mode? - lines = int64(0) + lines = int64(0) ) logs, err := s.rc.LeaseLogs(ctx, id, services, true, &lines) From eabf8058ebda2fc0f5a6a2724e72ff4746c97f5e Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Tue, 30 Apr 2024 23:51:50 -0600 Subject: [PATCH 09/17] feat: Finish tests and wire up client --- cmd/provider-services/cmd/leaseStatus.go | 96 ++++++++- cmd/provider-services/cmd/run.go | 2 + gateway/grpc/lease.go | 138 +++++++++++- gateway/grpc/server.go | 22 +- gateway/grpc/server_test.go | 264 ++++++++++++++++++++++- 5 files changed, 496 insertions(+), 26 deletions(-) diff --git a/cmd/provider-services/cmd/leaseStatus.go b/cmd/provider-services/cmd/leaseStatus.go index 954fc4240..65e1b53c2 100644 --- a/cmd/provider-services/cmd/leaseStatus.go +++ b/cmd/provider-services/cmd/leaseStatus.go @@ -1,17 +1,26 @@ package cmd import ( + "context" "crypto/tls" + "fmt" + "time" sdkclient "github.com/cosmos/cosmos-sdk/client" "github.com/spf13/cobra" + "github.com/akash-network/akash-api/go/manifest/v2beta2" + ptypes "github.com/akash-network/akash-api/go/node/provider/v1beta3" + leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" cmdcommon "github.com/akash-network/node/cmd/common" cutils "github.com/akash-network/node/x/cert/utils" dcli "github.com/akash-network/node/x/deployment/client/cli" mcli "github.com/akash-network/node/x/market/client/cli" + cltypes "github.com/akash-network/provider/cluster/types/v1beta3" + aclient "github.com/akash-network/provider/client" + gwgrpc "github.com/akash-network/provider/gateway/grpc" gwrest "github.com/akash-network/provider/gateway/rest" ) @@ -44,7 +53,7 @@ func doLeaseStatus(cmd *cobra.Command) error { return err } - prov, err := providerFromFlags(cmd.Flags()) + provAddr, err := providerFromFlags(cmd.Flags()) if err != nil { return err } @@ -59,15 +68,90 @@ func doLeaseStatus(cmd *cobra.Command) error { return markRPCServerError(err) } - gclient, err := gwrest.NewClient(cl, prov, []tls.Certificate{cert}) + prov, err := cl.Provider(ctx, &ptypes.QueryProviderRequest{Owner: provAddr.String()}) if err != nil { - return err + return fmt.Errorf("query client provider: %w", err) } - result, err := gclient.LeaseStatus(cmd.Context(), bid.LeaseID()) + hostURIgRPC, err := grpcURI(prov.GetProvider().HostURI) if err != nil { - return showErrorToUser(err) + return fmt.Errorf("grpc uri: %w", err) + } + + ctxDial, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var leaseStatus gwrest.LeaseStatus + + client, err := gwgrpc.NewClient(ctxDial, hostURIgRPC, cert, cl) + if err == nil { + res, err := client.ServiceStatus(ctx, &leasev1.ServiceStatusRequest{ + LeaseId: bid.LeaseID(), + }) + if err != nil { + return fmt.Errorf("service status: %w", err) + } + + leaseStatus = toLeaseStatus(res) + } else { + gclient, err := gwrest.NewClient(cl, provAddr, []tls.Certificate{cert}) + if err != nil { + return err + } + + leaseStatus, err = gclient.LeaseStatus(cmd.Context(), bid.LeaseID()) + if err != nil { + return showErrorToUser(err) + } + + } + + return cmdcommon.PrintJSON(cctx, leaseStatus) +} + +func toLeaseStatus(r *leasev1.ServiceStatusResponse) gwrest.LeaseStatus { + s := gwrest.LeaseStatus{ + Services: make(map[string]*cltypes.ServiceStatus), + } + + for _, svc := range r.Services { + s.Services[svc.Name] = &cltypes.ServiceStatus{ + Name: svc.Name, + Available: svc.Status.Available, + Total: svc.Status.Total, + URIs: svc.Status.Uris, + ObservedGeneration: svc.Status.ObservedGeneration, + Replicas: svc.Status.Replicas, + UpdatedReplicas: svc.Status.UpdatedReplicas, + ReadyReplicas: svc.Status.ReadyReplicas, + AvailableReplicas: svc.Status.AvailableReplicas, + } + + if len(svc.Ports) > 0 { + s.ForwardedPorts = make(map[string][]cltypes.ForwardedPortStatus) + } + for _, fp := range svc.Ports { + s.ForwardedPorts[svc.Name] = append(s.ForwardedPorts[svc.Name], cltypes.ForwardedPortStatus{ + Host: fp.Host, + Port: uint16(fp.Port), + ExternalPort: uint16(fp.ExternalPort), + Proto: v2beta2.ServiceProtocol(fp.Proto), + Name: fp.Name, + }) + } + + if len(svc.Ips) > 0 { + s.IPs = make(map[string][]gwrest.LeasedIPStatus) + } + for _, ip := range svc.Ips { + s.IPs[svc.Name] = append(s.IPs[svc.Name], gwrest.LeasedIPStatus{ + Port: ip.Port, + ExternalPort: ip.Port, + Protocol: ip.Protocol, + IP: ip.Ip, + }) + } } - return cmdcommon.PrintJSON(cctx, result) + return s } diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 523c5ee68..834363f7f 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -752,6 +752,8 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { gwgrpc.WithCerts([]tls.Certificate{tlsCert}), gwgrpc.WithProviderClient(service), gwgrpc.WithClusterReadClient(service.Cluster()), + gwgrpc.WithClusterSettings(clusterSettings), + gwgrpc.WithIPClient(ipOperatorClient), ) if err = gs.ServeOn(ctx, grpcaddr); err != nil { diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index d4105eb1c..267a07e83 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -8,8 +8,16 @@ import ( "sync" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" - "github.com/akash-network/provider/cluster/types/v1beta3" + kubeErrors "k8s.io/apimachinery/pkg/api/errors" + + kubeclienterrors "github.com/akash-network/provider/cluster/kube/errors" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" + "github.com/akash-network/provider/cluster/types/v1beta3/clients/ip" + crd "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" + "github.com/akash-network/provider/tools/fromctx" ) var ErrNoRunningPods = errors.New("no running pods") @@ -22,8 +30,122 @@ func (*server) ServiceLogs(context.Context, *leasev1.ServiceLogsRequest) (*lease panic("unimplemented") } -func (*server) ServiceStatus(context.Context, *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { - panic("unimplemented") +func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { + if s.ip == nil { + return &leasev1.ServiceStatusResponse{}, nil + } + + ctx = fromctx.ApplyToContext(ctx, s.clusterSettings) + + id := r.GetLeaseId() + found, m, err := s.rc.GetManifestGroup(ctx, id) + if err != nil { + return nil, fmt.Errorf("get manifest groups: %w", err) + } + + if !found { + return nil, status.Error(codes.NotFound, "lease does not exist") + } + + i := getInfo(m) + + ips := make(map[string][]leasev1.LeaseIPStatus) + if i.hasLeasedIPs { + var statuses []ip.LeaseIPStatus + if statuses, err = s.ip.GetIPAddressStatus(ctx, id.OrderID()); err != nil { + return nil, fmt.Errorf("get ip address status: %w", err) + } + + for _, stat := range statuses { + ips[stat.ServiceName] = append(ips[stat.ServiceName], leasev1.LeaseIPStatus{ + Port: stat.Port, + ExternalPort: stat.ExternalPort, + Protocol: stat.Protocol, + Ip: stat.IP, + }) + } + } + + ports := make(map[string][]leasev1.ForwarderPortStatus) + if i.hasForwardedPorts { + var allStatuses map[string][]ctypes.ForwardedPortStatus + if allStatuses, err = s.rc.ForwardedPortStatus(ctx, id); err != nil { + return nil, fmt.Errorf("forward port status: %w", err) + } + + for name, statuses := range allStatuses { + for _, stat := range statuses { + ports[name] = append(ports[name], leasev1.ForwarderPortStatus{ + Name: stat.Name, + Host: stat.Host, + Port: uint32(stat.Port), + ExternalPort: uint32(stat.ExternalPort), + Proto: string(stat.Proto), + }) + } + } + } + + serviceStatus, err := s.rc.LeaseStatus(ctx, id) + switch { + case errors.Is(err, kubeclienterrors.ErrNoDeploymentForLease): + return nil, status.Error(codes.NotFound, "no deployment for lease") + case errors.Is(err, kubeclienterrors.ErrLeaseNotFound): + return nil, status.Error(codes.NotFound, "lease does not exist") + case kubeErrors.IsNotFound(err): + return nil, status.Error(codes.NotFound, "not found") + case err != nil: + return nil, fmt.Errorf("lease status: %w", err) + } + + var resp leasev1.ServiceStatusResponse + for _, svc := range m.Services { + name := svc.Name + ss, ok := serviceStatus[name] + if !ok { + continue + } + + resp.Services = append(resp.Services, leasev1.ServiceStatus{ + Name: name, + Ports: ports[name], + Ips: ips[name], + Status: leasev1.LeaseServiceStatus{ + Available: ss.Available, + Total: ss.Total, + Uris: ss.URIs, + ObservedGeneration: ss.ObservedGeneration, + Replicas: ss.Replicas, + UpdatedReplicas: ss.UpdatedReplicas, + ReadyReplicas: ss.ReadyReplicas, + AvailableReplicas: ss.AvailableReplicas, + }, + }) + } + + return &resp, nil +} + +type manifestGroupInfo struct { + hasLeasedIPs bool + hasForwardedPorts bool +} + +func getInfo(m crd.ManifestGroup) manifestGroupInfo { + var i manifestGroupInfo + + for _, s := range m.Services { + for _, e := range s.Expose { + if len(e.IP) > 0 { + i.hasLeasedIPs = true + } + if e.Global && e.ExternalPort != 80 { + i.hasForwardedPorts = true + } + } + } + + return i } func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.LeaseRPC_StreamServiceLogsServer) error { @@ -49,7 +171,7 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L ) for _, ll := range logs { - go func(l *v1beta3.ServiceLog) { + go func(l *ctypes.ServiceLog) { defer l.Stream.Close() defer func() { errs <- nil }() @@ -99,10 +221,6 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L return nil } -func (*server) StreamServiceStatus(*leasev1.ServiceStatusRequest, leasev1.LeaseRPC_StreamServiceStatusServer) error { - panic("unimplemented") -} - type safeLogStream struct { s leasev1.LeaseRPC_StreamServiceLogsServer mu sync.Mutex @@ -118,3 +236,7 @@ func (s *safeLogStream) Send(r *leasev1.ServiceLogsResponse) error { return nil } + +func (*server) StreamServiceStatus(*leasev1.ServiceStatusRequest, leasev1.LeaseRPC_StreamServiceStatusServer) error { + panic("unimplemented") +} diff --git a/gateway/grpc/server.go b/gateway/grpc/server.go index 249290249..02709dc5b 100644 --- a/gateway/grpc/server.go +++ b/gateway/grpc/server.go @@ -22,6 +22,7 @@ import ( "github.com/akash-network/provider" "github.com/akash-network/provider/cluster" + "github.com/akash-network/provider/cluster/types/v1beta3/clients/ip" "github.com/akash-network/provider/tools/fromctx" ) @@ -47,6 +48,9 @@ type server struct { certs []tls.Certificate providerClient provider.Client clusterReadClient cluster.ReadClient + ip ip.Client + + clusterSettings map[any]any } func (s *grpcServer) ServeOn(ctx context.Context, addr string) error { @@ -83,6 +87,8 @@ type serverOpts struct { certs []tls.Certificate providerClient provider.Client clusterReadClient cluster.ReadClient + clusterSettings map[any]any + ipClient ip.Client } type opt func(*serverOpts) @@ -99,6 +105,14 @@ func WithClusterReadClient(c cluster.ReadClient) opt { return func(so *serverOpts) { so.clusterReadClient = c } } +func WithClusterSettings(s map[any]any) opt { + return func(so *serverOpts) { so.clusterSettings = s } +} + +func WithIPClient(c ip.Client) opt { + return func(so *serverOpts) { so.ipClient = c } +} + func NewServer(ctx context.Context, opts ...opt) Server { var o serverOpts for _, opt := range opts { @@ -130,9 +144,11 @@ func NewServer(ctx context.Context, opts ...opt) Server { ) s := &server{ - ctx: ctx, - pc: o.providerClient, - rc: o.clusterReadClient, + ctx: ctx, + pc: o.providerClient, + rc: o.clusterReadClient, + clusterSettings: o.clusterSettings, + ip: o.ipClient, } providerv1.RegisterProviderRPCServer(g, s) diff --git a/gateway/grpc/server_test.go b/gateway/grpc/server_test.go index ea35f542d..052bf5adb 100644 --- a/gateway/grpc/server_test.go +++ b/gateway/grpc/server_test.go @@ -20,13 +20,17 @@ import ( types "github.com/akash-network/akash-api/go/node/cert/v1beta3" qmock "github.com/akash-network/akash-api/go/node/client/v1beta2/mocks" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" + v1 "github.com/akash-network/akash-api/go/provider/lease/v1" providerv1 "github.com/akash-network/akash-api/go/provider/v1" "github.com/akash-network/node/testutil" cmocks "github.com/akash-network/provider/cluster/mocks" "github.com/akash-network/provider/cluster/types/v1beta3" - "github.com/akash-network/provider/mocks" + ctypes "github.com/akash-network/provider/cluster/types/v1beta3" + "github.com/akash-network/provider/cluster/types/v1beta3/clients/ip" + ipmocks "github.com/akash-network/provider/cluster/types/v1beta3/clients/ip/mocks" pmocks "github.com/akash-network/provider/mocks" + "github.com/akash-network/provider/pkg/apis/akash.network/v2beta2" ) type client struct { @@ -62,13 +66,15 @@ func TestRPCs(t *testing.T) { cases := []struct { desc string providerClient func() *pmocks.Client - readClient func() *cmocks.ReadClient + readClient func(t *testing.T) *cmocks.ReadClient + ipClient func(t *testing.T) *ipmocks.Client run func(context.Context, *testing.T, client) }{ + // GetStatus { desc: "GetStatus", - providerClient: func() *mocks.Client { - var m mocks.Client + providerClient: func() *pmocks.Client { + var m pmocks.Client m.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) return &m }, @@ -77,9 +83,146 @@ func TestRPCs(t *testing.T) { assert.NoError(t, err) }, }, + + // ServiceStatus + { + desc: "ServiceStatus get manifest error", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + + m.EXPECT().GetManifestGroup(mock.Anything, mock.Anything). + Return(false, v2beta2.ManifestGroup{}, errors.New("boom")) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + _, err := c.l.ServiceStatus(ctx, &leasev1.ServiceStatusRequest{}) + assert.ErrorContains(t, err, "boom") + }, + }, + { + desc: "ServiceStatus no manifest group", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + + m.EXPECT().GetManifestGroup(mock.Anything, mock.Anything). + Return(false, v2beta2.ManifestGroup{}, nil) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + _, err := c.l.ServiceStatus(ctx, &leasev1.ServiceStatusRequest{}) + assert.ErrorContains(t, err, "lease does not exist") + }, + }, + { + desc: "ServiceStatus", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + + mgi := newManifestGroup(t) + mgi.Services[0].Expose[0].IP = "1.2.3.4" + mgi.Services[0].Expose[0].Global = true + mgi.Services[0].Expose[0].ExternalPort = 8111 + + m.EXPECT().GetManifestGroup(mock.Anything, mock.Anything). + Return(true, mgi, nil) + + m.EXPECT().ForwardedPortStatus(mock.Anything, mock.Anything). + Return(map[string][]ctypes.ForwardedPortStatus{ + serviceName(t): { + { + Host: "host", + Port: 1111, + ExternalPort: 1112, + Proto: "test", + Name: serviceName(t), + }, + }, + }, nil) + + m.EXPECT().LeaseStatus(mock.Anything, mock.Anything). + Return(map[string]*ctypes.ServiceStatus{ + serviceName(t): { + Name: serviceName(t), + Available: 111, + Total: 222, + URIs: []string{"1", "2", "3"}, + ObservedGeneration: 1, + Replicas: 2, + UpdatedReplicas: 3, + ReadyReplicas: 4, + AvailableReplicas: 4, + }, + }, nil) + + return &m + }, + ipClient: func(t *testing.T) *ipmocks.Client { + var m ipmocks.Client + + m.EXPECT().GetIPAddressStatus(mock.Anything, mock.Anything). + Return([]ip.LeaseIPStatus{ + { + Port: 3333, + ExternalPort: 3334, + ServiceName: serviceName(t), + IP: "4.5.6.7", + Protocol: "test", + }, + }, nil) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + s, err := c.l.ServiceStatus(ctx, &leasev1.ServiceStatusRequest{ + Services: []string{serviceName(t)}, + }) + require.NoError(t, err) + + expected := v1.ServiceStatusResponse{ + Services: []v1.ServiceStatus{ + { + Name: serviceName(t), + Status: v1.LeaseServiceStatus{ + Available: 111, + Total: 222, + Uris: []string{"1", "2", "3"}, + ObservedGeneration: 1, + Replicas: 2, + UpdatedReplicas: 3, + ReadyReplicas: 4, + AvailableReplicas: 4, + }, + Ports: []v1.ForwarderPortStatus{ + { + Host: "host", + Port: 1111, + ExternalPort: 1112, + Proto: "test", + Name: serviceName(t), + }, + }, + Ips: []v1.LeaseIPStatus{ + { + Port: 3333, + ExternalPort: 3334, + Protocol: "test", + Ip: "4.5.6.7", + }, + }, + }, + }, + } + + assert.Equal(t, &expected, s) + }, + }, + + // StreamServiceLogs { desc: "StreamServiceLogs none", - readClient: func() *cmocks.ReadClient { + readClient: func(t *testing.T) *cmocks.ReadClient { var m cmocks.ReadClient m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return([]*v1beta3.ServiceLog{}, nil) @@ -95,7 +238,7 @@ func TestRPCs(t *testing.T) { }, { desc: "StreamServiceLogs LeaseLogs err", - readClient: func() *cmocks.ReadClient { + readClient: func(t *testing.T) *cmocks.ReadClient { var m cmocks.ReadClient m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(nil, errors.New("boom")) @@ -111,7 +254,7 @@ func TestRPCs(t *testing.T) { }, { desc: "StreamServiceLogs one", - readClient: func() *cmocks.ReadClient { + readClient: func(t *testing.T) *cmocks.ReadClient { var m cmocks.ReadClient stream := io.NopCloser(strings.NewReader("1\n2\n3")) @@ -149,7 +292,7 @@ func TestRPCs(t *testing.T) { }, { desc: "StreamServiceLogs multiple", - readClient: func() *cmocks.ReadClient { + readClient: func(t *testing.T) *cmocks.ReadClient { var m cmocks.ReadClient var ( @@ -220,6 +363,7 @@ func TestRPCs(t *testing.T) { var ( pc *pmocks.Client rc *cmocks.ReadClient + ip *ipmocks.Client ) if c.providerClient != nil { @@ -227,14 +371,19 @@ func TestRPCs(t *testing.T) { defer pc.AssertExpectations(t) } if c.readClient != nil { - rc = c.readClient() + rc = c.readClient(t) defer rc.AssertExpectations(t) } + if c.ipClient != nil { + ip = c.ipClient(t) + defer ip.AssertExpectations(t) + } s := NewServer(ctx, WithCerts(crt1.Cert), WithProviderClient(pc), WithClusterReadClient(rc), + WithIPClient(ip), ).(*grpcServer) defer s.Stop() @@ -265,6 +414,103 @@ func TestRPCs(t *testing.T) { } } +func Test_getInfo(t *testing.T) { + cases := []struct { + desc string + manifest func(t *testing.T) v2beta2.ManifestGroup + mgi manifestGroupInfo + }{ + { + desc: "has leased ips", + manifest: func(t *testing.T) v2beta2.ManifestGroup { + mgi := newManifestGroup(t) + + mgi.Services[0].Expose[0].IP = "1.2.3.4" + + return mgi + }, + mgi: manifestGroupInfo{ + hasLeasedIPs: true, + }, + }, + { + desc: "has forwarded ports", + manifest: func(t *testing.T) v2beta2.ManifestGroup { + mgi := newManifestGroup(t) + + mgi.Services[0].Expose[0].Global = true + mgi.Services[0].Expose[0].ExternalPort = 8111 + + return mgi + }, + mgi: manifestGroupInfo{ + hasForwardedPorts: true, + }, + }, + } + + for _, c := range cases { + c := c + + t.Run(c.desc, func(t *testing.T) { + mgi := getInfo(c.manifest(t)) + assert.Equal(t, c.mgi, mgi) + }) + } +} + +func newManifestGroup(t *testing.T) v2beta2.ManifestGroup { + t.Helper() + + return v2beta2.ManifestGroup{ + Name: serviceName(t), + Services: []v2beta2.ManifestService{{ + Name: serviceName(t), + Image: t.Name() + "_image", + Args: nil, + Env: nil, + Resources: v2beta2.Resources{ + CPU: v2beta2.ResourceCPU{ + Units: 1000, + }, + Memory: v2beta2.ResourceMemory{ + Size: "3333", + }, + Storage: v2beta2.ResourceStorage{ + { + Name: "default", + Size: "4444", + }, + }, + }, + Count: 1, + Expose: []v2beta2.ManifestServiceExpose{{ + Port: 8080, + ExternalPort: 80, + Proto: "TCP", + Service: serviceName(t), + Global: true, + Hosts: []string{"hello.localhost"}, + HTTPOptions: v2beta2.ManifestServiceExposeHTTPOptions{ + MaxBodySize: 1, + ReadTimeout: 2, + SendTimeout: 3, + NextTries: 4, + NextTimeout: 5, + NextCases: nil, + }, + IP: "", + EndpointSequenceNumber: 1, + }}, + Params: nil, + }}, + } +} + +func serviceName(t *testing.T) string { + return t.Name() + "_service" +} + func TestMTLS(t *testing.T) { var ( qclient = &qmock.QueryClient{} From 9640ab9c5c8b6b904a8311bf488a22805d8c4bb5 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 00:45:14 -0600 Subject: [PATCH 10/17] feat: Add StreamServiceStatus --- gateway/grpc/lease.go | 38 +++++++++++++++++++++++++++-- gateway/grpc/server_test.go | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 267a07e83..275be3c7d 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -6,9 +6,11 @@ import ( "fmt" "strings" "sync" + "time" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" kubeErrors "k8s.io/apimachinery/pkg/api/errors" @@ -237,6 +239,38 @@ func (s *safeLogStream) Send(r *leasev1.ServiceLogsResponse) error { return nil } -func (*server) StreamServiceStatus(*leasev1.ServiceStatusRequest, leasev1.LeaseRPC_StreamServiceStatusServer) error { - panic("unimplemented") +const defaultServiceStatusInterval = 5 * time.Second + +func (s *server) StreamServiceStatus(r *leasev1.ServiceStatusRequest, strm leasev1.LeaseRPC_StreamServiceStatusServer) error { + var ( + ctx = strm.Context() + id = r.GetLeaseId() + ) + + interval := defaultServiceStatusInterval + + if md, ok := metadata.FromIncomingContext(ctx); ok { + i := md.Get("interval")[0] + + var err error + if interval, err = time.ParseDuration(i); err != nil { + return fmt.Errorf("parse duration %s: %w", i, err) + } + } + + t := time.NewTicker(interval) + + for { + select { + case <-ctx.Done(): + return nil + case <-t.C: + res, err := s.ServiceStatus(ctx, &leasev1.ServiceStatusRequest{LeaseId: id}) + if err != nil { + return fmt.Errorf("service status: %w", err) + } + + strm.Send(res) + } + } } diff --git a/gateway/grpc/server_test.go b/gateway/grpc/server_test.go index 052bf5adb..0d7c886b9 100644 --- a/gateway/grpc/server_test.go +++ b/gateway/grpc/server_test.go @@ -9,12 +9,14 @@ import ( "net" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/emptypb" types "github.com/akash-network/akash-api/go/node/cert/v1beta3" @@ -349,6 +351,52 @@ func TestRPCs(t *testing.T) { assert.Equal(t, expected, actual) }, }, + + // StreamServiceStatus + { + desc: "StreamServiceStatus", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + + mgi := newManifestGroup(t) + + m.EXPECT().GetManifestGroup(mock.Anything, mock.Anything). + Return(true, mgi, nil) + + m.EXPECT().LeaseStatus(mock.Anything, mock.Anything). + Return(nil, nil) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + interval := 500 * time.Millisecond + + ctx = metadata.AppendToOutgoingContext(ctx, "interval", interval.String()) + s, err := c.l.StreamServiceStatus(ctx, &leasev1.ServiceStatusRequest{}) + require.NoError(t, err) + + var ( + iterations = 3 + after = time.After(interval * time.Duration(iterations)) + hits int + ) + + for { + select { + case <-after: + assert.Equal(t, iterations, hits) + return + default: + _, err = s.Recv() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + hits++ + } + } + }, + }, } for _, c := range cases { From bb0b1a69aa7460cd53ee75f2b6e83126aebbcd15 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 01:20:51 -0600 Subject: [PATCH 11/17] fix: e2e test --- gateway/grpc/lease.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 275be3c7d..33b9f9133 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -33,10 +33,6 @@ func (*server) ServiceLogs(context.Context, *leasev1.ServiceLogsRequest) (*lease } func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { - if s.ip == nil { - return &leasev1.ServiceStatusResponse{}, nil - } - ctx = fromctx.ApplyToContext(ctx, s.clusterSettings) id := r.GetLeaseId() @@ -52,7 +48,7 @@ func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequ i := getInfo(m) ips := make(map[string][]leasev1.LeaseIPStatus) - if i.hasLeasedIPs { + if s.ip != nil && i.hasLeasedIPs { var statuses []ip.LeaseIPStatus if statuses, err = s.ip.GetIPAddressStatus(ctx, id.OrderID()); err != nil { return nil, fmt.Errorf("get ip address status: %w", err) From 63a35cb2e58e35d1124988b70bdba9c1af2e87b7 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 11:11:57 -0600 Subject: [PATCH 12/17] fix: Error code in test --- integration/migrate_hostname_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/migrate_hostname_test.go b/integration/migrate_hostname_test.go index b8fb48988..d47a969a0 100644 --- a/integration/migrate_hostname_test.go +++ b/integration/migrate_hostname_test.go @@ -263,7 +263,7 @@ func (s *E2EMigrateHostname) TestE2EMigrateHostname() { fmt.Sprintf("--%s=%s", flags.FlagHome, s.validator.ClientCtx.HomeDir), ) s.Require().Error(err) - s.Require().Contains(err.Error(), "remote server returned 404") + s.Require().Contains(err.Error(), "lease does not exist") s.Require().NotNil(cmdResult) // confirm hostname still reachable, on the hostname it was migrated to From 90125f02e18451222200690995fe62e5d2e86141 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 13:47:14 -0600 Subject: [PATCH 13/17] fix: Bad merge --- gateway/grpc/server.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/gateway/grpc/server.go b/gateway/grpc/server.go index d0cb0884c..971741044 100644 --- a/gateway/grpc/server.go +++ b/gateway/grpc/server.go @@ -18,7 +18,6 @@ import ( ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" providerv1 "github.com/akash-network/akash-api/go/provider/v1" - sdk "github.com/cosmos/cosmos-sdk/types" "github.com/akash-network/provider" "github.com/akash-network/provider/cluster" @@ -165,8 +164,6 @@ func mtlsInterceptor(cquery ctypes.QueryClient) grpc.UnaryServerInterceptor { certificates := mtls.State.PeerCertificates if len(certificates) > 0 { - cquery := QueryClientFromCtx(ctx) - owner, _, err := atls.ValidatePeerCertificates(ctx, cquery, certificates, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) if err != nil { return nil, err @@ -174,22 +171,6 @@ func mtlsInterceptor(cquery ctypes.QueryClient) grpc.UnaryServerInterceptor { ctx = ContextWithOwner(ctx, owner) } - - clientCertPool := x509.NewCertPool() - clientCertPool.AddCert(cert) - - opts := x509.VerifyOptions{ - Roots: clientCertPool, - CurrentTime: time.Now(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - MaxConstraintComparisions: 0, - } - - if _, err = cert.Verify(opts); err != nil { - return nil, fmt.Errorf("tls: unable to verify certificate: %w", err) - } - - ctx = ContextWithOwner(ctx, owner) } } From 59b74522b05ea2a592b8758cff2b112bb183537b Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 21:00:28 -0600 Subject: [PATCH 14/17] feat: ServiceLogs --- cmd/provider-services/cmd/leaseLogs.go | 2 +- cmd/provider-services/cmd/leaseStatus.go | 2 +- gateway/grpc/lease.go | 55 ++++++++++++---- gateway/grpc/server.go | 20 +++--- gateway/grpc/server_test.go | 82 +++++++++++++++++++++++- go.mod | 6 +- go.sum | 8 +-- 7 files changed, 143 insertions(+), 32 deletions(-) diff --git a/cmd/provider-services/cmd/leaseLogs.go b/cmd/provider-services/cmd/leaseLogs.go index dc9139c3b..0dd262f61 100644 --- a/cmd/provider-services/cmd/leaseLogs.go +++ b/cmd/provider-services/cmd/leaseLogs.go @@ -235,7 +235,7 @@ func (g leaseLogGetter) rest(ctx context.Context, leases []mtypes.LeaseID) { for _, lid := range leases { stream := result{lid: lid} prov, _ := sdk.AccAddressFromBech32(lid.Provider) - gclient, err := gwrest.NewClient(ctx, g.cl, prov, []tls.Certificate{cert}) + gclient, err := gwrest.NewClient(ctx, g.cl, prov, []tls.Certificate{g.cert}) if err == nil { stream.stream, stream.error = gclient.LeaseLogs(ctx, lid, g.svcs, g.follow, g.tailLines) } else { diff --git a/cmd/provider-services/cmd/leaseStatus.go b/cmd/provider-services/cmd/leaseStatus.go index fff16d413..2006595af 100644 --- a/cmd/provider-services/cmd/leaseStatus.go +++ b/cmd/provider-services/cmd/leaseStatus.go @@ -94,7 +94,7 @@ func doLeaseStatus(cmd *cobra.Command) error { leaseStatus = toLeaseStatus(res) } else { - gclient, err := gwrest.NewClient(ctx, cl, prov, []tls.Certificate{cert}) + gclient, err := gwrest.NewClient(ctx, cl, provAddr, []tls.Certificate{cert}) if err != nil { return err } diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 33b9f9133..3a9fbeca1 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -28,8 +28,37 @@ func (*server) SendManifest(context.Context, *leasev1.SendManifestRequest) (*lea panic("unimplemented") } -func (*server) ServiceLogs(context.Context, *leasev1.ServiceLogsRequest) (*leasev1.ServiceLogsResponse, error) { - panic("unimplemented") +func (s *server) ServiceLogs(ctx context.Context, r *leasev1.ServiceLogsRequest) (*leasev1.ServiceLogsResponse, error) { + var ( + id = r.GetLeaseId() + services = strings.Join(r.GetServices(), " ") + lines = int64(0) + ) + + logs, err := s.rc.LeaseLogs(ctx, id, services, true, &lines) + if err != nil { + return nil, fmt.Errorf("lease logs: %w", err) + } + + if len(logs) == 0 { + return nil, ErrNoRunningPods + } + + var resp leasev1.ServiceLogsResponse + for _, l := range logs { + defer l.Stream.Close() + + for l.Scanner.Scan() { + resp.Services = append(resp.Services, + newLeaseV1ServiceLogs(l.Name, l.Scanner.Bytes())) + } + + if err := l.Scanner.Err(); err != nil { + return nil, fmt.Errorf("%s scanner: %w", l.Name, err) + } + } + + return &resp, nil } func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequest) (*leasev1.ServiceStatusResponse, error) { @@ -178,19 +207,9 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L case <-ctx.Done(): return default: - var ( - buf = l.Scanner.Bytes() - logs = make([]byte, len(buf)) - ) - - copy(logs, buf) - resp := leasev1.ServiceLogsResponse{ Services: []*leasev1.ServiceLogs{ - { - Name: l.Name, - Logs: buf, - }, + newLeaseV1ServiceLogs(l.Name, l.Scanner.Bytes()), }, } @@ -219,6 +238,16 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L return nil } +func newLeaseV1ServiceLogs(name string, buf []byte) *leasev1.ServiceLogs { + logs := make([]byte, len(buf)) + copy(logs, buf) + + return &leasev1.ServiceLogs{ + Name: name, + Logs: buf, + } +} + type safeLogStream struct { s leasev1.LeaseRPC_StreamServiceLogsServer mu sync.Mutex diff --git a/gateway/grpc/server.go b/gateway/grpc/server.go index 971741044..793f29d60 100644 --- a/gateway/grpc/server.go +++ b/gateway/grpc/server.go @@ -158,19 +158,19 @@ func NewServer(ctx context.Context, opts ...opt) Server { } func mtlsInterceptor(cquery ctypes.QueryClient) grpc.UnaryServerInterceptor { - return func(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { + return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if p, ok := peer.FromContext(ctx); ok { if mtls, ok := p.AuthInfo.(credentials.TLSInfo); ok { - certificates := mtls.State.PeerCertificates - - if len(certificates) > 0 { - owner, _, err := atls.ValidatePeerCertificates(ctx, cquery, certificates, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) - if err != nil { - return nil, err - } - - ctx = ContextWithOwner(ctx, owner) + owner, _, err := atls.ValidatePeerCertificates( + ctx, + cquery, + mtls.State.PeerCertificates, + []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + if err != nil { + return nil, err } + + ctx = ContextWithOwner(ctx, owner) } } diff --git a/gateway/grpc/server_test.go b/gateway/grpc/server_test.go index 0d7c886b9..f738b2d5a 100644 --- a/gateway/grpc/server_test.go +++ b/gateway/grpc/server_test.go @@ -86,6 +86,86 @@ func TestRPCs(t *testing.T) { }, }, + // ServiceLogs + { + desc: "ServiceLogs none", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*v1beta3.ServiceLog{}, nil) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + _, err := c.l.ServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + assert.ErrorContains(t, err, ErrNoRunningPods.Error()) + }, + }, + { + desc: "ServiceLogs LeaseLogs err", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, errors.New("boom")) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + _, err := c.l.ServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + assert.ErrorContains(t, err, "boom") + }, + }, + { + desc: "ServiceLogs", + readClient: func(t *testing.T) *cmocks.ReadClient { + var m cmocks.ReadClient + + var ( + stream1 = io.NopCloser(strings.NewReader("1_1\n1_2\n1_3")) + stream2 = io.NopCloser(strings.NewReader("2_1\n2_2\n2_3")) + stream3 = io.NopCloser(strings.NewReader("3_1\n3_2\n3_3")) + ) + + m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return([]*v1beta3.ServiceLog{ + { + Name: "one", + Stream: stream1, + Scanner: bufio.NewScanner(stream1), + }, + { + Name: "two", + Stream: stream2, + Scanner: bufio.NewScanner(stream2), + }, + { + Name: "three", + Stream: stream3, + Scanner: bufio.NewScanner(stream3), + }, + }, nil) + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + resp, err := c.l.ServiceLogs(ctx, &leasev1.ServiceLogsRequest{}) + require.NoError(t, err) + + expected := leasev1.ServiceLogsResponse{ + Services: []*leasev1.ServiceLogs{ + {Name: "one", Logs: []byte("1_1")}, + {Name: "one", Logs: []byte("1_2")}, + {Name: "one", Logs: []byte("1_3")}, + {Name: "two", Logs: []byte("2_1")}, + {Name: "two", Logs: []byte("2_2")}, + {Name: "two", Logs: []byte("2_3")}, + {Name: "three", Logs: []byte("3_1")}, + {Name: "three", Logs: []byte("3_2")}, + {Name: "three", Logs: []byte("3_3")}, + }, + } + + assert.EqualValues(t, &expected, resp) + }, + }, + // ServiceStatus { desc: "ServiceStatus get manifest error", @@ -597,7 +677,7 @@ func TestMTLS(t *testing.T) { cert: func(*testing.T) tls.Certificate { return tls.Certificate{} }, - errContains: "empty chain", + errContains: "too many peer certificates", }, } diff --git a/go.mod b/go.mod index 1d548b894..9de744b06 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/akash-network/provider go 1.21 require ( - github.com/akash-network/akash-api v0.0.66 - github.com/akash-network/node v0.34.0 + github.com/akash-network/akash-api v0.0.67 + github.com/akash-network/node v0.34.1 github.com/avast/retry-go/v4 v4.5.0 github.com/blang/semver/v4 v4.0.0 github.com/boz/go-lifecycle v0.1.1 @@ -53,6 +53,8 @@ replace ( // use cosmos fork of keyring github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 + github.com/akash-network/akash-api => github.com/akash-network/akash-api v0.0.68-0.20240501222454-6db85d461499 + github.com/cosmos/ledger-cosmos-go => github.com/akash-network/ledger-go/cosmos v0.14.4 // Fix upstream GHSA-h395-qcrw-5vmq vulnerability. diff --git a/go.sum b/go.sum index 0b2f91444..59e6609fc 100644 --- a/go.sum +++ b/go.sum @@ -197,16 +197,16 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/akash-network/akash-api v0.0.66 h1:HGYbjLmnKj7hNIO2V7f6CiHJfZJzeOBCIV45gRo/AbY= -github.com/akash-network/akash-api v0.0.66/go.mod h1:PdOQGTCX3kLBoKHdbPF9pe5+vSLANaMJbgA04UE+OqY= +github.com/akash-network/akash-api v0.0.68-0.20240501222454-6db85d461499 h1:kDqzcaf5K0zblTG3FlrLcy6UdROUXEA0jzd0J3kUgg0= +github.com/akash-network/akash-api v0.0.68-0.20240501222454-6db85d461499/go.mod h1:hlloP8i0WLhFn8Gq5KkX8nwFjpsxr1QzV7OqmgFGG6U= github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0= github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo= github.com/akash-network/ledger-go v0.14.3/go.mod h1:NfsjfFvno9Kaq6mfpsKz4sqjnAVVEsVsnBJfKB4ueAs= github.com/akash-network/ledger-go/cosmos v0.14.4 h1:h3WiXmoKKs9wkj1LHcJ12cLjXXg6nG1fp+UQ5+wu/+o= github.com/akash-network/ledger-go/cosmos v0.14.4/go.mod h1:SjAfheQTE4rWk0ir+wjbOWxwj8nc8E4AZ08NdsvYG24= -github.com/akash-network/node v0.34.0 h1:qBLEJlMDs7hSt1skomdPAtTXIXiAAmezYob8V+gG5Ks= -github.com/akash-network/node v0.34.0/go.mod h1:EnqNTPmvkKK0CHO1SqyF5ozAPJXpgmyFpBGak+KcPDY= +github.com/akash-network/node v0.34.1 h1:5ky3Q1dgXgGkcZA0y0AjEshi3fL7bk76OqCeh5ecMTs= +github.com/akash-network/node v0.34.1/go.mod h1:lPxn9dDCAXXflq9o1bqRH4DsLOiQaXIwCnU0l/nOLUs= github.com/alecthomas/participle/v2 v2.0.0-alpha7 h1:cK4vjj0VSgb3lN1nuKA5F7dw+1s1pWBe5bx7nNCnN+c= github.com/alecthomas/participle/v2 v2.0.0-alpha7/go.mod h1:NumScqsC42o9x+dGj8/YqsIfhrIQjFEOFovxotbBirA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= From 72ffe2ed72b75d7ca9fbec7a986e4af4ee69d988 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 21:05:49 -0600 Subject: [PATCH 15/17] fix: Deps --- go.mod | 24 +++++++++++++++--------- go.sum | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9de744b06..ce2131b20 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/akash-network/provider go 1.21 require ( - github.com/akash-network/akash-api v0.0.67 + github.com/akash-network/akash-api v0.34.0 github.com/akash-network/node v0.34.1 github.com/avast/retry-go/v4 v4.5.0 github.com/blang/semver/v4 v4.0.0 @@ -39,7 +39,7 @@ require ( golang.org/x/net v0.24.0 golang.org/x/sync v0.7.0 google.golang.org/grpc v1.63.2 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.34.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.26.1 k8s.io/apimachinery v0.26.1 @@ -75,12 +75,15 @@ require ( cosmossdk.io/depinject v1.0.0-alpha.3 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect - github.com/99designs/keyring v1.2.1 // indirect + github.com/99designs/keyring v1.2.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/BurntSushi/toml v1.2.1 // indirect github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect github.com/DataDog/zstd v1.5.0 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect github.com/alessio/shellescape v1.4.1 // indirect @@ -89,6 +92,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -111,14 +115,14 @@ require ( github.com/cosmos/ibc-go/v4 v4.6.0 // indirect github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect - github.com/danieljoos/wincred v1.1.2 // indirect + github.com/danieljoos/wincred v1.2.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/dvsekhvalnov/jose2go v1.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/edwingeng/deque/v2 v2.1.1 // indirect github.com/emicklei/go-restful/v3 v3.10.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect @@ -177,7 +181,7 @@ require ( github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/iancoleman/strcase v0.2.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/improbable-eng/grpc-web v0.14.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jaypipes/pcidb v1.0.0 // indirect @@ -221,6 +225,8 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/pseudomuto/protoc-gen-doc v1.5.1 // indirect + github.com/pseudomuto/protokit v0.2.1 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect @@ -260,9 +266,9 @@ require ( golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect - google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index 59e6609fc..e88c22172 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,7 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -43,6 +44,7 @@ cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg= +cloud.google.com/go/monitoring v1.18.2 h1:nIQdZdf1/9M0cEmXSlLB2jMq/k3CRh9p3oUzS06VDG8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -308,6 +310,8 @@ github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9E github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= +github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -456,6 +460,8 @@ github.com/cucumber/common/messages/go/v17 v17.1.1/go.mod h1:bpGxb57tDE385Rb2Eoh github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= +github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -522,6 +528,8 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= +github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -1192,6 +1200,8 @@ github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/improbable-eng/grpc-web v0.14.1 h1:NrN4PY71A6tAz2sKDvC5JCauENWp0ykG8Oq1H3cpFvw= github.com/improbable-eng/grpc-web v0.14.1/go.mod h1:zEjGHa8DAlkoOXmswrNvhUGEYQA9UI7DhrGeHR1DMGU= @@ -1704,6 +1714,10 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/pseudomuto/protoc-gen-doc v1.5.1 h1:Ah259kcrio7Ix1Rhb6u8FCaOkzf9qRBqXnvAufg061w= +github.com/pseudomuto/protoc-gen-doc v1.5.1/go.mod h1:XpMKYg6zkcpgfpCfQ8GcWBDRtRxOmMR5w7pz4Xo+dYM= +github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZfrJdk= +github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rboyer/safeio v0.2.1 h1:05xhhdRNAdS3apYm7JRjOqngf4xruaW959jmRxGDuSU= @@ -2551,10 +2565,16 @@ google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+n google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6 h1:MTmrc2F5TZKDKXigcZetYkH04YwqtOPEQJwh4PPOgfk= +google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6/go.mod h1:2ROWwqCIx97Y7CSyp11xB8fori0wzvD6+gbacaf5c8I= google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 h1:DTJM0R8LECCgFeUwApvcEJHz85HLagW8uRENYxHh1ww= +google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -2608,6 +2628,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0aef1b6d729fffa3c1ad8fcbeed3c8dd07988b40 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Wed, 1 May 2024 21:11:46 -0600 Subject: [PATCH 16/17] fix: Deps again --- go.mod | 7 +------ go.sum | 33 +++++---------------------------- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index ce2131b20..11a3abfc7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/akash-network/provider go 1.21 require ( - github.com/akash-network/akash-api v0.34.0 + github.com/akash-network/akash-api v0.0.67 github.com/akash-network/node v0.34.1 github.com/avast/retry-go/v4 v4.5.0 github.com/blang/semver/v4 v4.0.0 @@ -81,9 +81,6 @@ require ( github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect github.com/DataDog/zstd v1.5.0 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver v1.5.0 // indirect - github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect github.com/alessio/shellescape v1.4.1 // indirect @@ -225,8 +222,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect - github.com/pseudomuto/protoc-gen-doc v1.5.1 // indirect - github.com/pseudomuto/protokit v0.2.1 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect diff --git a/go.sum b/go.sum index e88c22172..708ad5526 100644 --- a/go.sum +++ b/go.sum @@ -22,7 +22,6 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -31,20 +30,19 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= -cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= -cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= -cloud.google.com/go/monitoring v1.18.0/go.mod h1:c92vVBCeq/OB4Ioyo+NbN2U7tlg5ZH41PZcdvfc+Lcg= cloud.google.com/go/monitoring v1.18.2 h1:nIQdZdf1/9M0cEmXSlLB2jMq/k3CRh9p3oUzS06VDG8= +cloud.google.com/go/monitoring v1.18.2/go.mod h1:MuL95M6d9HtXQOaWP9JxhFZJKP+fdTF0Gt5xl4IDsew= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -308,7 +306,6 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -458,8 +455,6 @@ github.com/cucumber/common/gherkin/go/v22 v22.0.0/go.mod h1:3mJT10B2GGn3MvVPd3Fw github.com/cucumber/common/messages/go/v17 v17.1.1 h1:RNqopvIFyLWnKv0LfATh34SWBhXeoFTJnSrgm9cT/Ts= github.com/cucumber/common/messages/go/v17 v17.1.1/go.mod h1:bpGxb57tDE385Rb2EohgUadLkAbhoC4IyCFi89u/JQI= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= -github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/danieljoos/wincred v1.2.1 h1:dl9cBrupW8+r5250DYkYxocLeZ1Y4vB1kxgtjxw8GQs= github.com/danieljoos/wincred v1.2.1/go.mod h1:uGaFL9fDn3OLTvzCGulzE+SzjEe5NGlh5FdCcyfPwps= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -526,8 +521,6 @@ github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74/go.mod github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= -github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -1198,8 +1191,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= @@ -1714,10 +1705,6 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/pseudomuto/protoc-gen-doc v1.5.1 h1:Ah259kcrio7Ix1Rhb6u8FCaOkzf9qRBqXnvAufg061w= -github.com/pseudomuto/protoc-gen-doc v1.5.1/go.mod h1:XpMKYg6zkcpgfpCfQ8GcWBDRtRxOmMR5w7pz4Xo+dYM= -github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZfrJdk= -github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rboyer/safeio v0.2.1 h1:05xhhdRNAdS3apYm7JRjOqngf4xruaW959jmRxGDuSU= @@ -2325,7 +2312,6 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2563,16 +2549,10 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6 h1:MTmrc2F5TZKDKXigcZetYkH04YwqtOPEQJwh4PPOgfk= google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6/go.mod h1:2ROWwqCIx97Y7CSyp11xB8fori0wzvD6+gbacaf5c8I= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 h1:DTJM0R8LECCgFeUwApvcEJHz85HLagW8uRENYxHh1ww= google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -2626,8 +2606,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -2687,7 +2665,6 @@ gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= From 5c83fea98d0d64ceac436397fd453d64ee93e2a9 Mon Sep 17 00:00:00 2001 From: Andrew Hare Date: Sun, 5 May 2024 11:59:13 -0600 Subject: [PATCH 17/17] feat: Validate and WIBOY --- cmd/provider-services/cmd/run.go | 2 +- gateway/grpc/client.go | 55 ++---------- gateway/grpc/lease.go | 10 +-- gateway/grpc/provider.go | 68 ++++++++++++++ gateway/grpc/server.go | 26 +++--- gateway/grpc/server_test.go | 149 +++++++++++++++++++++++-------- go.mod | 2 +- go.sum | 4 +- 8 files changed, 207 insertions(+), 109 deletions(-) diff --git a/cmd/provider-services/cmd/run.go b/cmd/provider-services/cmd/run.go index 834363f7f..a30d0c3a9 100644 --- a/cmd/provider-services/cmd/run.go +++ b/cmd/provider-services/cmd/run.go @@ -751,7 +751,7 @@ func doRunCmd(ctx context.Context, cmd *cobra.Command, _ []string) error { gs := gwgrpc.NewServer(ctx, gwgrpc.WithCerts([]tls.Certificate{tlsCert}), gwgrpc.WithProviderClient(service), - gwgrpc.WithClusterReadClient(service.Cluster()), + gwgrpc.WithClusterClient(service.Cluster()), gwgrpc.WithClusterSettings(clusterSettings), gwgrpc.WithIPClient(ipOperatorClient), ) diff --git a/gateway/grpc/client.go b/gateway/grpc/client.go index 4639d3419..659a68cb9 100644 --- a/gateway/grpc/client.go +++ b/gateway/grpc/client.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "errors" "fmt" - "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -14,7 +13,7 @@ import ( ctypes "github.com/akash-network/akash-api/go/node/cert/v1beta3" leasev1 "github.com/akash-network/akash-api/go/provider/lease/v1" providerv1 "github.com/akash-network/akash-api/go/provider/v1" - sdk "github.com/cosmos/cosmos-sdk/types" + atls "github.com/akash-network/akash-api/go/util/tls" ) type Client struct { @@ -42,58 +41,18 @@ func NewClient(ctx context.Context, addr string, cert tls.Certificate, cquery ct return errors.New("tls: invalid certificate chain") } - c, err := x509.ParseCertificate(chain[0]) + cert, err := x509.ParseCertificate(chain[0]) if err != nil { return fmt.Errorf("x509 parse certificate: %w", err) } - // validation - owner, err := sdk.AccAddressFromBech32(c.Subject.CommonName) - if err != nil { - return fmt.Errorf("tls: invalid certificate's subject common name: %w", err) - } - - // 1. CommonName in issuer and Subject must match and be as Bech32 format - if c.Subject.CommonName != c.Issuer.CommonName { - return fmt.Errorf("tls: invalid certificate's issuer common name: %w", err) - } - - // 2. serial number must be in - if c.SerialNumber == nil { - return fmt.Errorf("tls: invalid certificate serial number: %w", err) - } - - // 3. look up certificate on chain - var resp *ctypes.QueryCertificatesResponse - resp, err = cquery.Certificates( + _, _, err = atls.ValidatePeerCertificates( ctx, - &ctypes.QueryCertificatesRequest{ - Filter: ctypes.CertificateFilter{ - Owner: owner.String(), - Serial: c.SerialNumber.String(), - State: "valid", - }, - }, - ) + cquery, + []*x509.Certificate{cert}, + []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) if err != nil { - return fmt.Errorf("tls: unable to fetch certificate from chain: %w", err) - } - if (len(resp.Certificates) != 1) || !resp.Certificates[0].Certificate.IsState(ctypes.CertificateValid) { - return fmt.Errorf("tls: attempt to use non-existing or revoked certificate: %w", err) - } - - clientCertPool := x509.NewCertPool() - clientCertPool.AddCert(c) - - opts := x509.VerifyOptions{ - Roots: clientCertPool, - CurrentTime: time.Now(), - KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, - MaxConstraintComparisions: 0, - } - - if _, err = c.Verify(opts); err != nil { - return fmt.Errorf("tls: unable to verify certificate: %w", err) + return fmt.Errorf("validate peer certificates: %w", err) } return nil diff --git a/gateway/grpc/lease.go b/gateway/grpc/lease.go index 3a9fbeca1..833ec7a10 100644 --- a/gateway/grpc/lease.go +++ b/gateway/grpc/lease.go @@ -35,7 +35,7 @@ func (s *server) ServiceLogs(ctx context.Context, r *leasev1.ServiceLogsRequest) lines = int64(0) ) - logs, err := s.rc.LeaseLogs(ctx, id, services, true, &lines) + logs, err := s.cc.LeaseLogs(ctx, id, services, true, &lines) if err != nil { return nil, fmt.Errorf("lease logs: %w", err) } @@ -65,7 +65,7 @@ func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequ ctx = fromctx.ApplyToContext(ctx, s.clusterSettings) id := r.GetLeaseId() - found, m, err := s.rc.GetManifestGroup(ctx, id) + found, m, err := s.cc.GetManifestGroup(ctx, id) if err != nil { return nil, fmt.Errorf("get manifest groups: %w", err) } @@ -96,7 +96,7 @@ func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequ ports := make(map[string][]leasev1.ForwarderPortStatus) if i.hasForwardedPorts { var allStatuses map[string][]ctypes.ForwardedPortStatus - if allStatuses, err = s.rc.ForwardedPortStatus(ctx, id); err != nil { + if allStatuses, err = s.cc.ForwardedPortStatus(ctx, id); err != nil { return nil, fmt.Errorf("forward port status: %w", err) } @@ -113,7 +113,7 @@ func (s *server) ServiceStatus(ctx context.Context, r *leasev1.ServiceStatusRequ } } - serviceStatus, err := s.rc.LeaseStatus(ctx, id) + serviceStatus, err := s.cc.LeaseStatus(ctx, id) switch { case errors.Is(err, kubeclienterrors.ErrNoDeploymentForLease): return nil, status.Error(codes.NotFound, "no deployment for lease") @@ -183,7 +183,7 @@ func (s *server) StreamServiceLogs(r *leasev1.ServiceLogsRequest, strm leasev1.L lines = int64(0) ) - logs, err := s.rc.LeaseLogs(ctx, id, services, true, &lines) + logs, err := s.cc.LeaseLogs(ctx, id, services, true, &lines) if err != nil { return fmt.Errorf("lease logs: %w", err) } diff --git a/gateway/grpc/provider.go b/gateway/grpc/provider.go index 7a15585e0..1e6941e20 100644 --- a/gateway/grpc/provider.go +++ b/gateway/grpc/provider.go @@ -1,12 +1,16 @@ package grpc import ( + "fmt" + "runtime/debug" + providerv1 "github.com/akash-network/akash-api/go/provider/v1" "golang.org/x/net/context" "google.golang.org/protobuf/types/known/emptypb" "github.com/akash-network/provider/tools/fromctx" ptypes "github.com/akash-network/provider/types" + "github.com/akash-network/provider/version" ) func (s *server) GetStatus(ctx context.Context, _ *emptypb.Empty) (*providerv1.Status, error) { @@ -35,3 +39,67 @@ func (s *server) StreamStatus(_ *emptypb.Empty, stream providerv1.ProviderRPC_St } } } + +func (s *server) GetVersion(ctx context.Context, _ *emptypb.Empty) (*providerv1.GetVersionResponse, error) { + v := version.NewInfo() + + k, err := s.cc.KubeVersion() + if err != nil { + return nil, fmt.Errorf("kube version: %w", err) + } + + bd := make([]*providerv1.BuildDep, 0, len(v.BuildDeps)) + for _, b := range v.BuildDeps { + bd = append(bd, toProviderv1BuildDep(b.Module)) + } + + return &providerv1.GetVersionResponse{ + Akash: &providerv1.AkashInfo{ + Name: v.Name, + AppName: v.AppName, + Version: v.Version, + GitCommit: v.GitCommit, + BuildTags: v.BuildTags, + GoVersion: v.GoVersion, + BuildDeps: bd, + }, + Kube: &providerv1.KubeInfo{ + Major: k.Major, + Minor: k.Minor, + GitVersion: k.GitCommit, + GitTreeState: k.GitTreeState, + BuildDate: k.BuildDate, + GoVersion: k.GoVersion, + Compiler: k.Compiler, + Platform: k.Platform, + }, + }, nil +} + +func toProviderv1BuildDep(m *debug.Module) *providerv1.BuildDep { + if m == nil { + return nil + } + + return &providerv1.BuildDep{ + Path: m.Path, + Version: m.Version, + Sum: m.Sum, + Replace: toProviderv1BuildDep(m.Replace), + } +} + +func (s *server) Validate(ctx context.Context, r *providerv1.ValidateRequest) (*providerv1.ValidateResponse, error) { + v, err := s.pc.Validate(ctx, OwnerFromCtx(ctx), r.GetGroup()) + if err != nil { + return nil, fmt.Errorf("validate: %w", err) + } + + return &providerv1.ValidateResponse{ + MinBidPrice: v.MinBidPrice, + }, nil +} + +func (s *server) WIBOY(ctx context.Context, r *providerv1.ValidateRequest) (*providerv1.ValidateResponse, error) { + return s.Validate(ctx, r) +} diff --git a/gateway/grpc/server.go b/gateway/grpc/server.go index 793f29d60..c3d4c0dd5 100644 --- a/gateway/grpc/server.go +++ b/gateway/grpc/server.go @@ -42,12 +42,10 @@ type grpcServer struct { type server struct { ctx context.Context pc provider.Client - rc cluster.ReadClient + cc cluster.Client - certs []tls.Certificate - providerClient provider.Client - clusterReadClient cluster.ReadClient - ip ip.Client + certs []tls.Certificate + ip ip.Client clusterSettings map[any]any } @@ -83,11 +81,11 @@ func (s *grpcServer) ServeOn(ctx context.Context, addr string) error { } type serverOpts struct { - certs []tls.Certificate - providerClient provider.Client - clusterReadClient cluster.ReadClient - clusterSettings map[any]any - ipClient ip.Client + certs []tls.Certificate + providerClient provider.Client + clusterClient cluster.Client + clusterSettings map[any]any + ipClient ip.Client } type opt func(*serverOpts) @@ -100,8 +98,8 @@ func WithProviderClient(c provider.Client) opt { return func(so *serverOpts) { so.providerClient = c } } -func WithClusterReadClient(c cluster.ReadClient) opt { - return func(so *serverOpts) { so.clusterReadClient = c } +func WithClusterClient(c cluster.Client) opt { + return func(so *serverOpts) { so.clusterClient = c } } func WithClusterSettings(s map[any]any) opt { @@ -145,7 +143,7 @@ func NewServer(ctx context.Context, opts ...opt) Server { s := &server{ ctx: ctx, pc: o.providerClient, - rc: o.clusterReadClient, + cc: o.clusterClient, clusterSettings: o.clusterSettings, ip: o.ipClient, } @@ -167,7 +165,7 @@ func mtlsInterceptor(cquery ctypes.QueryClient) grpc.UnaryServerInterceptor { mtls.State.PeerCertificates, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) if err != nil { - return nil, err + return nil, fmt.Errorf("validate peer certificates: %w", err) } ctx = ContextWithOwner(ctx, owner) diff --git a/gateway/grpc/server_test.go b/gateway/grpc/server_test.go index f738b2d5a..8ea1439a8 100644 --- a/gateway/grpc/server_test.go +++ b/gateway/grpc/server_test.go @@ -26,6 +26,11 @@ import ( providerv1 "github.com/akash-network/akash-api/go/provider/v1" "github.com/akash-network/node/testutil" + "github.com/akash-network/provider" + + sdk "github.com/cosmos/cosmos-sdk/types" + kubeversion "k8s.io/apimachinery/pkg/version" + cmocks "github.com/akash-network/provider/cluster/mocks" "github.com/akash-network/provider/cluster/types/v1beta3" ctypes "github.com/akash-network/provider/cluster/types/v1beta3" @@ -67,15 +72,15 @@ func TestRPCs(t *testing.T) { cases := []struct { desc string - providerClient func() *pmocks.Client - readClient func(t *testing.T) *cmocks.ReadClient - ipClient func(t *testing.T) *ipmocks.Client + providerClient func(*testing.T) *pmocks.Client + clusterClient func(*testing.T) *cmocks.Client + ipClient func(*testing.T) *ipmocks.Client run func(context.Context, *testing.T, client) }{ // GetStatus { desc: "GetStatus", - providerClient: func() *pmocks.Client { + providerClient: func(t *testing.T) *pmocks.Client { var m pmocks.Client m.EXPECT().StatusV1(mock.Anything).Return(&providerv1.Status{}, nil) return &m @@ -86,11 +91,79 @@ func TestRPCs(t *testing.T) { }, }, + // GetVersion + { + desc: "GetVersion", + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client + + m.EXPECT().KubeVersion().Return(&kubeversion.Info{ + Major: "1", + Minor: "2", + GitVersion: "3", + GitTreeState: "4", + BuildDate: "5", + GoVersion: "6", + Compiler: "7", + Platform: "8", + }, nil) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + r, err := c.p.GetVersion(ctx, &emptypb.Empty{}) + require.NoError(t, err) + + assert.NotEmpty(t, r.Kube) + assert.NotEmpty(t, r.Akash) + }, + }, + + // Validate + { + desc: "Validate error", + providerClient: func(t *testing.T) *pmocks.Client { + var m pmocks.Client + + m.EXPECT().Validate(mock.Anything, mock.Anything, mock.Anything). + Return(provider.ValidateGroupSpecResult{}, errors.New("boom")) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + _, err := c.p.Validate(ctx, &providerv1.ValidateRequest{}) + assert.ErrorContains(t, err, "boom") + }, + }, + { + desc: "Validate", + providerClient: func(t *testing.T) *pmocks.Client { + var m pmocks.Client + + m.EXPECT().Validate(mock.Anything, mock.Anything, mock.Anything). + Return(provider.ValidateGroupSpecResult{ + MinBidPrice: sdk.DecCoin{ + Denom: t.Name(), + Amount: sdk.NewDec(111), + }, + }, nil) + + return &m + }, + run: func(ctx context.Context, t *testing.T, c client) { + res, err := c.p.Validate(ctx, &providerv1.ValidateRequest{}) + require.NoError(t, err) + + assert.Equal(t, t.Name(), res.MinBidPrice.Denom) + assert.Equal(t, sdk.NewDec(111), res.MinBidPrice.Amount) + }, + }, + // ServiceLogs { desc: "ServiceLogs none", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return([]*v1beta3.ServiceLog{}, nil) return &m @@ -102,8 +175,8 @@ func TestRPCs(t *testing.T) { }, { desc: "ServiceLogs LeaseLogs err", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(nil, errors.New("boom")) return &m @@ -115,8 +188,8 @@ func TestRPCs(t *testing.T) { }, { desc: "ServiceLogs", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client var ( stream1 = io.NopCloser(strings.NewReader("1_1\n1_2\n1_3")) @@ -169,8 +242,8 @@ func TestRPCs(t *testing.T) { // ServiceStatus { desc: "ServiceStatus get manifest error", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client m.EXPECT().GetManifestGroup(mock.Anything, mock.Anything). Return(false, v2beta2.ManifestGroup{}, errors.New("boom")) @@ -184,8 +257,8 @@ func TestRPCs(t *testing.T) { }, { desc: "ServiceStatus no manifest group", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client m.EXPECT().GetManifestGroup(mock.Anything, mock.Anything). Return(false, v2beta2.ManifestGroup{}, nil) @@ -199,8 +272,8 @@ func TestRPCs(t *testing.T) { }, { desc: "ServiceStatus", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client mgi := newManifestGroup(t) mgi.Services[0].Expose[0].IP = "1.2.3.4" @@ -304,8 +377,8 @@ func TestRPCs(t *testing.T) { // StreamServiceLogs { desc: "StreamServiceLogs none", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return([]*v1beta3.ServiceLog{}, nil) return &m @@ -320,8 +393,8 @@ func TestRPCs(t *testing.T) { }, { desc: "StreamServiceLogs LeaseLogs err", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client m.EXPECT().LeaseLogs(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(nil, errors.New("boom")) return &m @@ -336,8 +409,8 @@ func TestRPCs(t *testing.T) { }, { desc: "StreamServiceLogs one", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client stream := io.NopCloser(strings.NewReader("1\n2\n3")) @@ -374,8 +447,8 @@ func TestRPCs(t *testing.T) { }, { desc: "StreamServiceLogs multiple", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client var ( stream1 = io.NopCloser(strings.NewReader("1_1\n1_2\n1_3")) @@ -435,8 +508,8 @@ func TestRPCs(t *testing.T) { // StreamServiceStatus { desc: "StreamServiceStatus", - readClient: func(t *testing.T) *cmocks.ReadClient { - var m cmocks.ReadClient + clusterClient: func(t *testing.T) *cmocks.Client { + var m cmocks.Client mgi := newManifestGroup(t) @@ -490,17 +563,17 @@ func TestRPCs(t *testing.T) { var ( pc *pmocks.Client - rc *cmocks.ReadClient + cc *cmocks.Client ip *ipmocks.Client ) if c.providerClient != nil { - pc = c.providerClient() + pc = c.providerClient(t) defer pc.AssertExpectations(t) } - if c.readClient != nil { - rc = c.readClient(t) - defer rc.AssertExpectations(t) + if c.clusterClient != nil { + cc = c.clusterClient(t) + defer cc.AssertExpectations(t) } if c.ipClient != nil { ip = c.ipClient(t) @@ -510,7 +583,7 @@ func TestRPCs(t *testing.T) { s := NewServer(ctx, WithCerts(crt1.Cert), WithProviderClient(pc), - WithClusterReadClient(rc), + WithClusterClient(cc), WithIPClient(ip), ).(*grpcServer) @@ -663,19 +736,19 @@ func TestMTLS(t *testing.T) { cases := []struct { desc string - cert func(*testing.T) tls.Certificate + certs func() []tls.Certificate errContains string }{ { desc: "good cert", - cert: func(*testing.T) tls.Certificate { - return testutil.Certificate(t, testutil.AccAddress(t), com, cod).Cert[0] + certs: func() []tls.Certificate { + return crt.Cert }, }, { desc: "empty chain", - cert: func(*testing.T) tls.Certificate { - return tls.Certificate{} + certs: func() []tls.Certificate { + return nil }, errContains: "too many peer certificates", }, @@ -709,7 +782,7 @@ func TestMTLS(t *testing.T) { tlsConfig := tls.Config{ InsecureSkipVerify: true, - Certificates: []tls.Certificate{c.cert(t)}, + Certificates: c.certs(), } conn, err := grpc.DialContext(ctx, l.Addr().String(), diff --git a/go.mod b/go.mod index 11a3abfc7..9579bcbb2 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ replace ( // use cosmos fork of keyring github.com/99designs/keyring => github.com/cosmos/keyring v1.2.0 - github.com/akash-network/akash-api => github.com/akash-network/akash-api v0.0.68-0.20240501222454-6db85d461499 + github.com/akash-network/akash-api => github.com/akash-network/akash-api v0.0.68-0.20240505172348-f082c5cdbb68 github.com/cosmos/ledger-cosmos-go => github.com/akash-network/ledger-go/cosmos v0.14.4 diff --git a/go.sum b/go.sum index 708ad5526..3219e6e27 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,8 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/akash-network/akash-api v0.0.68-0.20240501222454-6db85d461499 h1:kDqzcaf5K0zblTG3FlrLcy6UdROUXEA0jzd0J3kUgg0= -github.com/akash-network/akash-api v0.0.68-0.20240501222454-6db85d461499/go.mod h1:hlloP8i0WLhFn8Gq5KkX8nwFjpsxr1QzV7OqmgFGG6U= +github.com/akash-network/akash-api v0.0.68-0.20240505172348-f082c5cdbb68 h1:wlTZfjQRVsRKWObgpTiBuUOqxPoQ2bKxVvmMuSDuYqA= +github.com/akash-network/akash-api v0.0.68-0.20240505172348-f082c5cdbb68/go.mod h1:VETP3ilOwLIRP40WPy806ygL/IzgGEuMoxXaCUZpRPI= github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0= github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=