diff --git a/README.md b/README.md index 44e7e81..79d0a47 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,44 @@ We recommend to use `nix` to fulfill all development dependencies. Visit [Nix Do - to build the binary run `nix-shell --run 'make build'` - to run tests run `nix-shell --run 'make test'` +## Configuration + +This gateway supports the same object storage configuration as the main Thanos project. You can use the `--objstore.config-file` or `--objstore.config` flags to specify your storage backend. + +### Supported Storage Backends + +Thanks to the integration with [thanos-io/objstore](https://github.com/thanos-io/objstore), the gateway supports all major cloud storage providers: + +- **Amazon S3** (and S3-compatible like MinIO) +- **Google Cloud Storage (GCS)** +- **Microsoft Azure Blob Storage** +- **OpenStack Swift** +- **Tencent Cloud Object Storage (COS)** +- **Alibaba Cloud Object Storage Service (OSS)** +- **Local Filesystem** + +### Configuration Examples + +See the [`examples/`](examples/) directory for configuration examples: + +- [`objstore-s3.yaml`](examples/objstore-s3.yaml) - Amazon S3 configuration +- [`objstore-gcs.yaml`](examples/objstore-gcs.yaml) - Google Cloud Storage configuration +- [`objstore-azure.yaml`](examples/objstore-azure.yaml) - Azure Blob Storage configuration +- [`objstore-filesystem.yaml`](examples/objstore-filesystem.yaml) - Local filesystem configuration +- [`objstore-minio.yaml`](examples/objstore-minio.yaml) - MinIO/S3-compatible configuration + +For complete configuration format details, see the [Thanos Storage Documentation](https://thanos.io/tip/thanos/storage.md/#configuration). + ## Running ### Server -Once built, you can run the server using something like: +Once built, you can run the server using: ```bash -parquet-gateway serve \ - --storage.prefix my-prefix \ +# Using config file (recommended) +thanos-parquet-gateway serve \ + --objstore.config-file=examples/objstore-s3.yaml \ --http.internal.port=6060 \ --http.prometheus.port=9090 \ --http.thanos.port=9091 \ @@ -31,11 +60,23 @@ parquet-gateway serve \ --block.discovery.concurrency=32 \ --query.external-label=prometheus=my-prometheus \ --query.external-label=replica=ha-1 + +# Using inline config +thanos-parquet-gateway serve \ + --objstore.config=" +type: S3 +config: + bucket: my-bucket + endpoint: s3.amazonaws.com +" \ + --http.internal.port=6060 \ + --http.prometheus.port=9090 \ + --http.thanos.port=9091 ``` This will: -- load blocks from the `.data/my-prefix` directory +- load blocks from your configured object storage - expose internal metrics and readiness handlers on port 6060 - expose a subset of the Prometheus HTTP API on port 9090 - expose an Thanos Info, Series and Query gRPC service on port 9091 @@ -62,19 +103,33 @@ curl 'http://0.0.0.0:9000/api/v1/query' \ ] } } - ``` ### Converter -To convert TSDB blocks in the `.data/source` directory that overlap `09/2021` and write the resulting parquet files into the `.data/destination` directory. +To convert TSDB blocks from one storage to another, you can specify separate configurations for source (TSDB) and destination (Parquet) storage: ```bash -parquet-gateway convert \ - --tsdb.storage.prefix source \ - --parquet.storage.prefix destination \ +# Using separate config files +thanos-parquet-gateway convert \ + --tsdb.objstore.config-file=tsdb-storage.yaml \ + --parquet.objstore.config-file=parquet-storage.yaml \ --convert.sorting.label=__name__ \ --convert.sorting.label=namespace + +# Using inline configs +thanos-parquet-gateway convert \ + --tsdb.objstore.config=" +type: S3 +config: + bucket: tsdb-source-bucket +" \ + --parquet.objstore.config=" +type: S3 +config: + bucket: parquet-destination-bucket +" \ + --convert.sorting.label=__name__ ``` ## Note diff --git a/cmd/config.go b/cmd/config.go index f95b539..06a2df6 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -10,11 +10,10 @@ import ( "log/slog" "net/http" "net/http/pprof" + "os" "strings" "time" - "gopkg.in/yaml.v3" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/jaeger" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" @@ -29,6 +28,7 @@ import ( "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/client" "github.com/thanos-io/thanos/pkg/runutil" + "gopkg.in/alecthomas/kingpin.v2" "github.com/thanos-io/thanos-parquet-gateway/locate" ) @@ -45,65 +45,42 @@ func setupInterrupt(ctx context.Context, g *run.Group, log *slog.Logger) { } type bucketOpts struct { - storage string - prefix string - - // filesystem options - filesystemDirectory string - - // s3 options - s3Bucket string - s3Endpoint string - s3AccessKey string - s3SecretKey string - s3Insecure bool + // Thanos-style configuration + configFile string + config string +} - retries int +// registerThanosStyleFlags registers the standard Thanos objstore flags +func (opts *bucketOpts) registerThanosStyleFlags(cmd *kingpin.CmdClause, prefix string) { + configFlagSuffix := ".objstore.config" + configFlag := strings.TrimPrefix(prefix+configFlagSuffix, ".") + cmd.Flag(configFlag, "Alternative to 'objstore.config-file' flag (mutually exclusive). Content of YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration"). + PlaceHolder("").StringVar(&opts.config) + + configFileFlagSuffix := ".objstore.config-file" + configFileFlag := strings.TrimPrefix(prefix+configFileFlagSuffix, ".") + cmd.Flag(configFileFlag, "Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration"). + PlaceHolder("").StringVar(&opts.configFile) } func setupBucket(log *slog.Logger, opts bucketOpts) (objstore.Bucket, error) { - prov := objstore.ObjProvider(strings.ToUpper(opts.storage)) - cfg := client.BucketConfig{ - Type: prov, - Prefix: opts.prefix, - } - var subCfg any - switch prov { - case objstore.FILESYSTEM: - subCfg = struct { - Directory string `yaml:"directory"` - }{ - Directory: opts.filesystemDirectory, - } - case objstore.S3: - subCfg = struct { - Bucket string `yaml:"bucket"` - Endpoint string `yaml:"endpoint"` - AccessKey string `yaml:"access_key"` - SecretKey string `yaml:"secret_key"` - MaxRetries int `yaml:"max_retries"` - Insecure bool `yaml:"insecure"` - }{ - Bucket: opts.s3Bucket, - Endpoint: opts.s3Endpoint, - AccessKey: opts.s3AccessKey, - SecretKey: opts.s3SecretKey, - Insecure: opts.s3Insecure, - MaxRetries: opts.retries, - } - default: - return nil, fmt.Errorf("unknown bucket type: %s", prov) - } + var confContentYaml []byte + var err error - cfg.Config = subCfg - bytes, err := yaml.Marshal(cfg) - if err != nil { - return nil, fmt.Errorf("unable to marshal bucket config yaml: %w", err) + if opts.configFile != "" { + confContentYaml, err = os.ReadFile(opts.configFile) + if err != nil { + return nil, fmt.Errorf("reading config file: %w", err) + } + } else if opts.config != "" { + confContentYaml = []byte(opts.config) + } else { + return nil, fmt.Errorf("no objstore configuration provided. Use --objstore.config-file or --objstore.config") } - bkt, err := client.NewBucket(slogAdapter{log}, bytes, "parquet-gateway", nil) + bkt, err := client.NewBucket(slogAdapter{log}, confContentYaml, "thanos-parquet-gateway", nil) if err != nil { - return nil, fmt.Errorf("unable to create bucket client: %w", err) + return nil, fmt.Errorf("creating bucket client: %w", err) } return bkt, nil diff --git a/cmd/convert.go b/cmd/convert.go index 827f431..fb13e0d 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -79,25 +79,13 @@ func (opts *conversionOpts) registerFlags(cmd *kingpin.CmdClause) { } func (opts *bucketOpts) registerConvertParquetFlags(cmd *kingpin.CmdClause) { - cmd.Flag("parquet.storage.type", "type of storage").Default("filesystem").EnumVar(&opts.storage, "filesystem", "s3") - cmd.Flag("parquet.storage.prefix", "prefix for the storage").Default("").StringVar(&opts.prefix) - cmd.Flag("parquet.storage.filesystem.directory", "directory for filesystem").Default(".data").StringVar(&opts.filesystemDirectory) - cmd.Flag("parquet.storage.s3.bucket", "bucket for s3").Default("").StringVar(&opts.s3Bucket) - cmd.Flag("parquet.storage.s3.endpoint", "endpoint for s3").Default("").StringVar(&opts.s3Endpoint) - cmd.Flag("parquet.storage.s3.access_key", "access key for s3").Default("").Envar("PARQUET_STORAGE_S3_ACCESS_KEY").StringVar(&opts.s3AccessKey) - cmd.Flag("parquet.storage.s3.secret_key", "secret key for s3").Default("").Envar("PARQUET_STORAGE_S3_SECRET_KEY").StringVar(&opts.s3SecretKey) - cmd.Flag("parquet.storage.s3.insecure", "use http").Default("false").BoolVar(&opts.s3Insecure) + // Add Thanos-style flags + opts.registerThanosStyleFlags(cmd, "parquet") } func (opts *bucketOpts) registerConvertTSDBFlags(cmd *kingpin.CmdClause) { - cmd.Flag("tsdb.storage.type", "type of storage").Default("filesystem").EnumVar(&opts.storage, "filesystem", "s3") - cmd.Flag("tsdb.storage.prefix", "prefix for the storage").Default("").StringVar(&opts.prefix) - cmd.Flag("tsdb.storage.filesystem.directory", "directory for filesystem").Default(".data").StringVar(&opts.filesystemDirectory) - cmd.Flag("tsdb.storage.s3.bucket", "bucket for s3").Default("").StringVar(&opts.s3Bucket) - cmd.Flag("tsdb.storage.s3.endpoint", "endpoint for s3").Default("").StringVar(&opts.s3Endpoint) - cmd.Flag("tsdb.storage.s3.access_key", "access key for s3").Default("").Envar("TSDB_STORAGE_S3_ACCESS_KEY").StringVar(&opts.s3AccessKey) - cmd.Flag("tsdb.storage.s3.secret_key", "secret key for s3").Default("").Envar("TSDB_STORAGE_S3_SECRET_KEY").StringVar(&opts.s3SecretKey) - cmd.Flag("tsdb.storage.s3.insecure", "use http").Default("false").BoolVar(&opts.s3Insecure) + // Add Thanos-style flags + opts.registerThanosStyleFlags(cmd, "tsdb") } func (opts *discoveryOpts) registerConvertParquetFlags(cmd *kingpin.CmdClause) { diff --git a/cmd/serve.go b/cmd/serve.go index 775d596..13e60c0 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -63,14 +63,8 @@ func (opts *serveOpts) registerFlags(cmd *kingpin.CmdClause) { } func (opts *bucketOpts) registerServeFlags(cmd *kingpin.CmdClause) { - cmd.Flag("storage.type", "type of storage").Default("filesystem").EnumVar(&opts.storage, "filesystem", "s3") - cmd.Flag("storage.prefix", "prefix for the storage").Default("").StringVar(&opts.prefix) - cmd.Flag("storage.filesystem.directory", "directory for filesystem").Default(".data").StringVar(&opts.filesystemDirectory) - cmd.Flag("storage.s3.bucket", "bucket for s3").Default("").StringVar(&opts.s3Bucket) - cmd.Flag("storage.s3.endpoint", "endpoint for s3").Default("").StringVar(&opts.s3Endpoint) - cmd.Flag("storage.s3.access_key", "access key for s3").Default("").Envar("STORAGE_S3_ACCESS_KEY").StringVar(&opts.s3AccessKey) - cmd.Flag("storage.s3.secret_key", "secret key for s3").Default("").Envar("STORAGE_S3_SECRET_KEY").StringVar(&opts.s3SecretKey) - cmd.Flag("storage.s3.insecure", "use http").Default("false").BoolVar(&opts.s3Insecure) + // Add Thanos-style flags + opts.registerThanosStyleFlags(cmd, "") } func (opts *tracingOpts) registerServeFlags(cmd *kingpin.CmdClause) { diff --git a/examples/objstore-azure.yaml b/examples/objstore-azure.yaml new file mode 100644 index 0000000..bcaee56 --- /dev/null +++ b/examples/objstore-azure.yaml @@ -0,0 +1,7 @@ +type: AZURE +config: + storage_account: "" + storage_account_key: "" + container: "thanos" + endpoint: "" + max_retries: 0 diff --git a/examples/objstore-filesystem.yaml b/examples/objstore-filesystem.yaml new file mode 100644 index 0000000..be70681 --- /dev/null +++ b/examples/objstore-filesystem.yaml @@ -0,0 +1,3 @@ +type: FILESYSTEM +config: + directory: "./data" diff --git a/examples/objstore-gcs.yaml b/examples/objstore-gcs.yaml new file mode 100644 index 0000000..2daafed --- /dev/null +++ b/examples/objstore-gcs.yaml @@ -0,0 +1,4 @@ +type: GCS +config: + bucket: "my-thanos-bucket" + service_account: "" diff --git a/examples/objstore-minio.yaml b/examples/objstore-minio.yaml new file mode 100644 index 0000000..b2b1e2a --- /dev/null +++ b/examples/objstore-minio.yaml @@ -0,0 +1,11 @@ +type: S3 +config: + bucket: "thanos" + endpoint: "minio.example.com:9000" + access_key: "minio" + secret_key: "minio123" + insecure: true + signature_version2: false + http_config: + idle_conn_timeout: 90s + response_header_timeout: 2m diff --git a/examples/objstore-s3.yaml b/examples/objstore-s3.yaml new file mode 100644 index 0000000..b7b7253 --- /dev/null +++ b/examples/objstore-s3.yaml @@ -0,0 +1,15 @@ +type: S3 +config: + bucket: "my-thanos-bucket" + endpoint: "s3.amazonaws.com" + region: "us-east-1" + access_key: "" + secret_key: "" + insecure: false + signature_version2: false + put_user_metadata: {} + http_config: + idle_conn_timeout: 90s + response_header_timeout: 2m + trace: + enable: false