diff --git a/controller/.env b/controller/.env index 6548374..b9e7f11 100644 --- a/controller/.env +++ b/controller/.env @@ -1,3 +1,6 @@ SERVER_HOST=localhost SERVER_PORT=50051 -MAX_MESSAGE_SIZE=4194304 \ No newline at end of file +MAX_MESSAGE_SIZE=4194304 +REDIS_PASSWORD=mysecretpassword +REDIS_USER=myuser +REDIS_USER_PASSWORD=myuserpassword \ No newline at end of file diff --git a/controller/MODULE.bazel b/controller/MODULE.bazel index 7cc9134..af07aa2 100644 --- a/controller/MODULE.bazel +++ b/controller/MODULE.bazel @@ -24,8 +24,13 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_joho_godotenv", + "com_github_subosito_gotenv", "com_github_pelletier_go_toml", "com_github_stretchr_testify", "org_golang_google_grpc", "org_golang_google_protobuf", -) + # Добавляем Redis в use_repo + "com_github_redis_go_redis_v9", + "com_github_cespare_xxhash_v2", + "com_github_dgryski_go_rendezvous", +) \ No newline at end of file diff --git a/controller/cmd/grpc_server/BUILD b/controller/cmd/grpc_server/BUILD index 309a12a..cbe6ae7 100644 --- a/controller/cmd/grpc_server/BUILD +++ b/controller/cmd/grpc_server/BUILD @@ -1,14 +1,5 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") -go_binary( - name = "grpc_server", - embed = [":grpc_server_lib"], - visibility = ["//visibility:public"], - data = [ - "//internal/service/config", - ], -) - go_library( name = "grpc_server_lib", srcs = ["main.go"], @@ -19,9 +10,17 @@ go_library( "//internal/grpcserver", "//internal/manager", "//internal/service/service", + "//internal/service/storage", "//pkg/proto/admin_service:admin_service_go_proto", "//pkg/proto/communication:communication_go_proto", "@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//reflection", + "@com_github_subosito_gotenv//:go_default_library", ], ) + +go_binary( + name = "grpc_server", + embed = [":grpc_server_lib"], + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/controller/cmd/grpc_server/main.go b/controller/cmd/grpc_server/main.go index 821b4ff..b7a1c28 100644 --- a/controller/cmd/grpc_server/main.go +++ b/controller/cmd/grpc_server/main.go @@ -1,18 +1,41 @@ package main import ( + "fmt" "log" "net" + "os" "github.com/moevm/grpc_server/internal/config" "github.com/moevm/grpc_server/internal/grpcserver" "github.com/moevm/grpc_server/internal/manager" + "github.com/moevm/grpc_server/internal/service/storage" adminPb "github.com/moevm/grpc_server/pkg/proto/admin_service" commPb "github.com/moevm/grpc_server/pkg/proto/communication" + "github.com/subosito/gotenv" "google.golang.org/grpc" "google.golang.org/grpc/reflection" ) +func init() { + err := gotenv.Load() + if err != nil { + log.Fatalf("Error to load env: %v", err) + } +} + +func LoadConfigRedis() (storage.Config, error) { + addr, exist := os.LookupEnv("REDIS_ADDR") + + if !exist { + return storage.Config{}, fmt.Errorf("REDIS_ADDR does not exists") + } + + return storage.Config{ + Addr: addr, + }, nil +} + func main() { cfg := config.Load() adminServer := grpcserver.NewAdminServer() @@ -23,7 +46,13 @@ func main() { adminServer.SetManager(mgr) - dataServer, err := grpcserver.NewDataServer(mgr, "internal/service/config/categories.json", "internal/service/config/providers.json") + configRedis, err := LoadConfigRedis() + + if err != nil { + log.Fatalf("Error to load redis config: %v", err) + } + + dataServer, err := grpcserver.NewDataServer(mgr, "internal/service/config/categories.json", "internal/service/config/providers.json", configRedis) if err != nil { log.Fatalf("Failed to create data server: %v", err) diff --git a/controller/docker-compose.yaml b/controller/docker-compose.yaml new file mode 100644 index 0000000..d53a072 --- /dev/null +++ b/controller/docker-compose.yaml @@ -0,0 +1,50 @@ +version: '3.9' + +services: + redis: + image: redis:latest + container_name: redis_container + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_USER=${REDIS_USER} + - REDIS_USER_PASSWORD=${REDIS_USER_PASSWORD} + ports: + - "6379:6379" + volumes: + - ./redisdata:/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + reservations: + cpus: '0.25' + memory: 256M + command: > + sh -c ' + mkdir -p /usr/local/etc/redis && + echo "bind 0.0.0.0" > /usr/local/etc/redis/redis.conf && + echo "requirepass $REDIS_PASSWORD" >> /usr/local/etc/redis/redis.conf && + echo "appendonly yes" >> /usr/local/etc/redis/redis.conf && + echo "appendfsync everysec" >> /usr/local/etc/redis/redis.conf && + echo "user default on nopass ~* +@all" > /usr/local/etc/redis/users.acl && + echo "user $REDIS_USER on >$REDIS_USER_PASSWORD ~* +@all" >> /usr/local/etc/redis/users.acl && + redis-server /usr/local/etc/redis/redis.conf --aclfile /usr/local/etc/redis/users.acl + ' + healthcheck: + test: ["CMD", "redis-cli", "-a", "$REDIS_PASSWORD", "ping"] + interval: 30s + timeout: 10s + retries: 5 + restart: unless-stopped + tty: true + stdin_open: true + + controller: + build: + dockerfile: Dockerfile.controller + environment: + - REDIS_ADDR=redis:6379 + depends_on: + - redis + diff --git a/controller/go.mod b/controller/go.mod index a442c6e..4923cf0 100644 --- a/controller/go.mod +++ b/controller/go.mod @@ -11,8 +11,13 @@ require ( ) require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/redis/go-redis/v9 v9.18.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect diff --git a/controller/go.sum b/controller/go.sum index 9f76e60..ebca917 100644 --- a/controller/go.sum +++ b/controller/go.sum @@ -1,5 +1,9 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -12,12 +16,17 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/nacknime-official/fsm-telebot-redis-storage/v3 v3.0.0-20250404091544-e45a7caa1738/go.mod h1:VmEWTtxh9kff1N+ZcaoappPqjitkzRAGXWLogBwsdHM= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= @@ -30,6 +39,8 @@ go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= diff --git a/controller/internal/grpcserver/BUILD b/controller/internal/grpcserver/BUILD index f16d9e1..d46f634 100644 --- a/controller/internal/grpcserver/BUILD +++ b/controller/internal/grpcserver/BUILD @@ -11,13 +11,14 @@ go_library( deps = [ "//internal/service/service", "//internal/manager", + "//internal/service/storage", "//pkg/proto/admin_service:admin_service_go_proto", "//pkg/proto/communication:communication_go_proto", + "@com_github_redis_go_redis_v9//:go-redis", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", "@org_golang_google_grpc//:go_default_library", - "@org_golang_google_protobuf//types/known/emptypb", + "@org_golang_google_protobuf//types/known/emptypb", "@org_golang_google_protobuf//proto:go_default_library", - ] -) - + ], +) \ No newline at end of file diff --git a/controller/internal/grpcserver/data_server.go b/controller/internal/grpcserver/data_server.go index 15ddc7e..08c0552 100644 --- a/controller/internal/grpcserver/data_server.go +++ b/controller/internal/grpcserver/data_server.go @@ -3,8 +3,10 @@ package grpcserver import ( "context" "log" + "time" "github.com/moevm/grpc_server/internal/manager" + "github.com/moevm/grpc_server/internal/service/storage" pb "github.com/moevm/grpc_server/pkg/proto/communication" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -18,18 +20,30 @@ type DataServer struct { pb.UnimplementedDataServiceServer manager *manager.Manager classifier *service.Service + storage *storage.RedisClient } -func NewDataServer(mgr *manager.Manager, categoryFile, providerFile string) (*DataServer, error) { +func NewDataServer(mgr *manager.Manager, categoryFile, providerFile string, config storage.Config) (*DataServer, error) { classifier, err := service.NewService(categoryFile, providerFile) if err != nil { return nil, err } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + + defer cancel() + + redis, err := storage.NewRedisClient(ctx, config) + + if err != nil { + log.Fatalf("Error to create redis storage: %v", err) + } + return &DataServer{ manager: mgr, classifier: classifier, + storage: redis, }, nil } @@ -73,13 +87,31 @@ func (s *DataServer) Classify(ctx context.Context, req *pb.ClassifyRequest) (*pb log.Printf("gRPC Classify from worker %d: type=%s, target=%s", req.WorkerId, req.Type, req.Target) - categoryIDs, err := s.classifier.Check(req.Target, req.Type) + cashRequest, err := s.storage.GetRequestHash(ctx, req.Domain) + + var categoryIDs []int + if err != nil { - log.Printf("Classification error: %v", err) - return &pb.ClassifyResponse{ - Categories: []string{"unknown"}, - TrustLevel: 0, - }, nil + categoryIDs, err = s.classifier.Check(req.Domain, "domain") + if err != nil { + log.Printf("Classification error: %v", err) + return &pb.ClassifyResponse{ + Categories: []string{"unknown"}, + TrustLevel: 0, + }, nil + } + + err = s.storage.SaveRequestHash(ctx, storage.RequestCash{ + Endpoint: req.Domain, + CategoriesIds: categoryIDs, + }) + + if err != nil { + log.Printf("Error save hash: %v", err.Error()) + } + + } else { + categoryIDs = cashRequest.CategoriesIds } if len(categoryIDs) > 0 { diff --git a/controller/internal/service/storage/BUILD b/controller/internal/service/storage/BUILD new file mode 100644 index 0000000..8345881 --- /dev/null +++ b/controller/internal/service/storage/BUILD @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "storage", + srcs = [ + "redis.go" + ], + importpath = "github.com/moevm/grpc_server/internal/service/storage", + visibility = ["//visibility:public"], + deps = [ + "@com_github_redis_go_redis_v9//:go-redis", + ], +) \ No newline at end of file diff --git a/controller/internal/service/storage/redis.go b/controller/internal/service/storage/redis.go new file mode 100644 index 0000000..6f144bb --- /dev/null +++ b/controller/internal/service/storage/redis.go @@ -0,0 +1,89 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +type Config struct { + Addr string + Password string + User string + DB int + MaxRetries int + DialTimeout time.Duration + Timeout time.Duration +} + +type RedisClient struct { + db *redis.Client +} + +type RequestCash struct { + Endpoint string + CategoriesIds []int +} + +func NewRedisClient(ctx context.Context, cfg Config) (*RedisClient, error) { + db := redis.NewClient(&redis.Options{ + Addr: cfg.Addr, + Password: cfg.Password, + DB: cfg.DB, + Username: cfg.User, + MaxRetries: cfg.MaxRetries, + DialTimeout: cfg.DialTimeout, + ReadTimeout: cfg.Timeout, + WriteTimeout: cfg.Timeout, + }) + + if err := db.Ping(ctx).Err(); err != nil { + fmt.Printf("failed to connect to redis server: %s\n", err.Error()) + return nil, err + } + + return &RedisClient{db: db}, nil +} + +func (c *RedisClient) SaveRequestHash(ctx context.Context, hash RequestCash) error { + + key := fmt.Sprintf("request:hash:%s", hash.Endpoint) + + data, err := json.Marshal(hash.CategoriesIds) + + if err != nil { + return err + } + + return c.db.Set(ctx, key, data, 0).Err() +} + +func (c *RedisClient) GetRequestHash(ctx context.Context, endpoint string) (*RequestCash, error) { + + key := fmt.Sprintf("request:hash:%s", endpoint) + + cmd := c.db.Get(ctx, key) + + if cmd.Err() != nil { + return nil, cmd.Err() + } + + data, err := cmd.Bytes() + + if err != nil { + return nil, err + } + + var categoriesID []int + + err = json.Unmarshal(data, &categoriesID) + + if err != nil { + return nil, err + } + + return &RequestCash{Endpoint: endpoint, CategoriesIds: categoriesID}, nil +}