feat: add thanos style storage config

Signed-off-by: Joel Verezhak <[email protected]>

Author: verejoel

README.md

@@ -13,15 +13,44 @@ We recommend to use `nix` to fulfill all development dependencies. Visit [Nix Do
 - to build the binary run `nix-shell --run 'make build'`
 - to run tests run `nix-shell --run 'make test'`
 
+## Configuration
+
+This gateway supports the same object storage configuration as the main Thanos project. You can use the `--objstore.config-file` or `--objstore.config` flags to specify your storage backend.
+
+### Supported Storage Backends
+
+Thanks to the integration with [thanos-io/objstore](https://github.com/thanos-io/objstore), the gateway supports all major cloud storage providers:
+
+- **Amazon S3** (and S3-compatible like MinIO)
+- **Google Cloud Storage (GCS)**
+- **Microsoft Azure Blob Storage**
+- **OpenStack Swift**
+- **Tencent Cloud Object Storage (COS)**
+- **Alibaba Cloud Object Storage Service (OSS)**
+- **Local Filesystem**
+
+### Configuration Examples
+
+See the [`examples/`](examples/) directory for configuration examples:
+
+- [`objstore-s3.yaml`](examples/objstore-s3.yaml) - Amazon S3 configuration
+- [`objstore-gcs.yaml`](examples/objstore-gcs.yaml) - Google Cloud Storage configuration  
+- [`objstore-azure.yaml`](examples/objstore-azure.yaml) - Azure Blob Storage configuration
+- [`objstore-filesystem.yaml`](examples/objstore-filesystem.yaml) - Local filesystem configuration
+- [`objstore-minio.yaml`](examples/objstore-minio.yaml) - MinIO/S3-compatible configuration
+
+For complete configuration format details, see the [Thanos Storage Documentation](https://thanos.io/tip/thanos/storage.md/#configuration).
+
 ## Running
 
 ### Server
 
-Once built, you can run the server using something like:
+Once built, you can run the server using:
 
 ```bash
-parquet-gateway serve \
-    --storage.prefix my-prefix \
+# Using config file (recommended)
+thanos-parquet-gateway serve \
+    --objstore.config-file=examples/objstore-s3.yaml \
     --http.internal.port=6060 \
     --http.prometheus.port=9090 \
     --http.thanos.port=9091 \
@@ -31,11 +60,23 @@ parquet-gateway serve \
     --block.discovery.concurrency=32 \
     --query.external-label=prometheus=my-prometheus \
     --query.external-label=replica=ha-1
+
+# Using inline config
+thanos-parquet-gateway serve \
+    --objstore.config="
+type: S3
+config:
+  bucket: my-bucket
+  endpoint: s3.amazonaws.com
+" \
+    --http.internal.port=6060 \
+    --http.prometheus.port=9090 \
+    --http.thanos.port=9091
 ```
 
 This will:
 
-- load blocks from the `.data/my-prefix` directory
+- load blocks from your configured object storage
 - expose internal metrics and readiness handlers on port 6060
 - expose a subset of the Prometheus HTTP API on port 9090
 - expose an Thanos Info, Series and Query gRPC service on port 9091
@@ -62,19 +103,33 @@ curl 'http://0.0.0.0:9000/api/v1/query' \
     ]
   }
 }
-
 ```
 
 ### Converter
 
-To convert TSDB blocks in the `.data/source` directory that overlap `09/2021` and write the resulting parquet files into the `.data/destination` directory.
+To convert TSDB blocks from one storage to another, you can specify separate configurations for source (TSDB) and destination (Parquet) storage:
 
 ```bash
-parquet-gateway convert \
-    --tsdb.storage.prefix source \
-    --parquet.storage.prefix destination \
+# Using separate config files
+thanos-parquet-gateway convert \
+    --tsdb.objstore.config-file=tsdb-storage.yaml \
+    --parquet.objstore.config-file=parquet-storage.yaml \
     --convert.sorting.label=__name__ \
     --convert.sorting.label=namespace
+
+# Using inline configs
+thanos-parquet-gateway convert \
+    --tsdb.objstore.config="
+type: S3
+config:
+  bucket: tsdb-source-bucket
+" \
+    --parquet.objstore.config="
+type: S3  
+config:
+  bucket: parquet-destination-bucket
+" \
+    --convert.sorting.label=__name__
 ```
 
 ## Note

cmd/config.go

@@ -10,11 +10,10 @@ import (
 	"log/slog"
 	"net/http"
 	"net/http/pprof"
+	"os"
 	"strings"
 	"time"
 
-	"gopkg.in/yaml.v3"
-
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
@@ -29,6 +28,7 @@ import (
 	"github.com/thanos-io/objstore"
 	"github.com/thanos-io/objstore/client"
 	"github.com/thanos-io/thanos/pkg/runutil"
+	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/thanos-io/thanos-parquet-gateway/locate"
 )
@@ -45,65 +45,42 @@ func setupInterrupt(ctx context.Context, g *run.Group, log *slog.Logger) {
 }
 
 type bucketOpts struct {
-	storage string
-	prefix  string
-
-	// filesystem options
-	filesystemDirectory string
-
-	// s3 options
-	s3Bucket    string
-	s3Endpoint  string
-	s3AccessKey string
-	s3SecretKey string
-	s3Insecure  bool
+	// Thanos-style configuration
+	configFile string
+	config     string
+}
 
-	retries int
+// registerThanosStyleFlags registers the standard Thanos objstore flags
+func (opts *bucketOpts) registerThanosStyleFlags(cmd *kingpin.CmdClause, prefix string) {
+	configFlagSuffix := ".objstore.config"
+	configFlag := strings.TrimPrefix(prefix+configFlagSuffix, ".")
+	cmd.Flag(configFlag, "Alternative to 'objstore.config-file' flag (mutually exclusive). Content of YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration").
+		PlaceHolder("<content>").StringVar(&opts.config)
+
+	configFileFlagSuffix := ".objstore.config-file"
+	configFileFlag := strings.TrimPrefix(prefix+configFileFlagSuffix, ".")
+	cmd.Flag(configFileFlag, "Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration").
+		PlaceHolder("<file-path>").StringVar(&opts.configFile)
 }
 
 func setupBucket(log *slog.Logger, opts bucketOpts) (objstore.Bucket, error) {
-	prov := objstore.ObjProvider(strings.ToUpper(opts.storage))
-	cfg := client.BucketConfig{
-		Type:   prov,
-		Prefix: opts.prefix,
-	}
-	var subCfg any
-	switch prov {
-	case objstore.FILESYSTEM:
-		subCfg = struct {
-			Directory string `yaml:"directory"`
-		}{
-			Directory: opts.filesystemDirectory,
-		}
-	case objstore.S3:
-		subCfg = struct {
-			Bucket     string `yaml:"bucket"`
-			Endpoint   string `yaml:"endpoint"`
-			AccessKey  string `yaml:"access_key"`
-			SecretKey  string `yaml:"secret_key"`
-			MaxRetries int    `yaml:"max_retries"`
-			Insecure   bool   `yaml:"insecure"`
-		}{
-			Bucket:     opts.s3Bucket,
-			Endpoint:   opts.s3Endpoint,
-			AccessKey:  opts.s3AccessKey,
-			SecretKey:  opts.s3SecretKey,
-			Insecure:   opts.s3Insecure,
-			MaxRetries: opts.retries,
-		}
-	default:
-		return nil, fmt.Errorf("unknown bucket type: %s", prov)
-	}
+	var confContentYaml []byte
+	var err error
 
-	cfg.Config = subCfg
-	bytes, err := yaml.Marshal(cfg)
-	if err != nil {
-		return nil, fmt.Errorf("unable to marshal bucket config yaml: %w", err)
+	if opts.configFile != "" {
+		confContentYaml, err = os.ReadFile(opts.configFile)
+		if err != nil {
+			return nil, fmt.Errorf("reading config file: %w", err)
+		}
+	} else if opts.config != "" {
+		confContentYaml = []byte(opts.config)
+	} else {
+		return nil, fmt.Errorf("no objstore configuration provided. Use --objstore.config-file or --objstore.config")
 	}
 
-	bkt, err := client.NewBucket(slogAdapter{log}, bytes, "parquet-gateway", nil)
+	bkt, err := client.NewBucket(slogAdapter{log}, confContentYaml, "thanos-parquet-gateway", nil)
 	if err != nil {
-		return nil, fmt.Errorf("unable to create bucket client: %w", err)
+		return nil, fmt.Errorf("creating bucket client: %w", err)
 	}
 
 	return bkt, nil

cmd/convert.go

@@ -79,25 +79,13 @@ func (opts *conversionOpts) registerFlags(cmd *kingpin.CmdClause) {
 }
 
 func (opts *bucketOpts) registerConvertParquetFlags(cmd *kingpin.CmdClause) {
-	cmd.Flag("parquet.storage.type", "type of storage").Default("filesystem").EnumVar(&opts.storage, "filesystem", "s3")
-	cmd.Flag("parquet.storage.prefix", "prefix for the storage").Default("").StringVar(&opts.prefix)
-	cmd.Flag("parquet.storage.filesystem.directory", "directory for filesystem").Default(".data").StringVar(&opts.filesystemDirectory)
-	cmd.Flag("parquet.storage.s3.bucket", "bucket for s3").Default("").StringVar(&opts.s3Bucket)
-	cmd.Flag("parquet.storage.s3.endpoint", "endpoint for s3").Default("").StringVar(&opts.s3Endpoint)
-	cmd.Flag("parquet.storage.s3.access_key", "access key for s3").Default("").Envar("PARQUET_STORAGE_S3_ACCESS_KEY").StringVar(&opts.s3AccessKey)
-	cmd.Flag("parquet.storage.s3.secret_key", "secret key for s3").Default("").Envar("PARQUET_STORAGE_S3_SECRET_KEY").StringVar(&opts.s3SecretKey)
-	cmd.Flag("parquet.storage.s3.insecure", "use http").Default("false").BoolVar(&opts.s3Insecure)
+	// Add Thanos-style flags
+	opts.registerThanosStyleFlags(cmd, "parquet")
 }
 
 func (opts *bucketOpts) registerConvertTSDBFlags(cmd *kingpin.CmdClause) {
-	cmd.Flag("tsdb.storage.type", "type of storage").Default("filesystem").EnumVar(&opts.storage, "filesystem", "s3")
-	cmd.Flag("tsdb.storage.prefix", "prefix for the storage").Default("").StringVar(&opts.prefix)
-	cmd.Flag("tsdb.storage.filesystem.directory", "directory for filesystem").Default(".data").StringVar(&opts.filesystemDirectory)
-	cmd.Flag("tsdb.storage.s3.bucket", "bucket for s3").Default("").StringVar(&opts.s3Bucket)
-	cmd.Flag("tsdb.storage.s3.endpoint", "endpoint for s3").Default("").StringVar(&opts.s3Endpoint)
-	cmd.Flag("tsdb.storage.s3.access_key", "access key for s3").Default("").Envar("TSDB_STORAGE_S3_ACCESS_KEY").StringVar(&opts.s3AccessKey)
-	cmd.Flag("tsdb.storage.s3.secret_key", "secret key for s3").Default("").Envar("TSDB_STORAGE_S3_SECRET_KEY").StringVar(&opts.s3SecretKey)
-	cmd.Flag("tsdb.storage.s3.insecure", "use http").Default("false").BoolVar(&opts.s3Insecure)
+	// Add Thanos-style flags
+	opts.registerThanosStyleFlags(cmd, "tsdb")
 }
 
 func (opts *discoveryOpts) registerConvertParquetFlags(cmd *kingpin.CmdClause) {

cmd/serve.go

@@ -63,14 +63,8 @@ func (opts *serveOpts) registerFlags(cmd *kingpin.CmdClause) {
 }
 
 func (opts *bucketOpts) registerServeFlags(cmd *kingpin.CmdClause) {
-	cmd.Flag("storage.type", "type of storage").Default("filesystem").EnumVar(&opts.storage, "filesystem", "s3")
-	cmd.Flag("storage.prefix", "prefix for the storage").Default("").StringVar(&opts.prefix)
-	cmd.Flag("storage.filesystem.directory", "directory for filesystem").Default(".data").StringVar(&opts.filesystemDirectory)
-	cmd.Flag("storage.s3.bucket", "bucket for s3").Default("").StringVar(&opts.s3Bucket)
-	cmd.Flag("storage.s3.endpoint", "endpoint for s3").Default("").StringVar(&opts.s3Endpoint)
-	cmd.Flag("storage.s3.access_key", "access key for s3").Default("").Envar("STORAGE_S3_ACCESS_KEY").StringVar(&opts.s3AccessKey)
-	cmd.Flag("storage.s3.secret_key", "secret key for s3").Default("").Envar("STORAGE_S3_SECRET_KEY").StringVar(&opts.s3SecretKey)
-	cmd.Flag("storage.s3.insecure", "use http").Default("false").BoolVar(&opts.s3Insecure)
+	// Add Thanos-style flags
+	opts.registerThanosStyleFlags(cmd, "")
 }
 
 func (opts *tracingOpts) registerServeFlags(cmd *kingpin.CmdClause) {

examples/objstore-azure.yaml

@@ -0,0 +1,7 @@
+type: AZURE
+config:
+  storage_account: ""
+  storage_account_key: ""
+  container: "thanos"
+  endpoint: ""
+  max_retries: 0

examples/objstore-filesystem.yaml

@@ -0,0 +1,3 @@
+type: FILESYSTEM
+config:
+  directory: "./data"

examples/objstore-gcs.yaml

@@ -0,0 +1,4 @@
+type: GCS
+config:
+  bucket: "my-thanos-bucket"
+  service_account: ""

examples/objstore-minio.yaml

@@ -0,0 +1,11 @@
+type: S3
+config:
+  bucket: "thanos"
+  endpoint: "minio.example.com:9000"
+  access_key: "minio"
+  secret_key: "minio123"
+  insecure: true
+  signature_version2: false
+  http_config:
+    idle_conn_timeout: 90s
+    response_header_timeout: 2m

examples/objstore-minio.yml

@@ -0,0 +1,11 @@
+type: S3
+config:
+  bucket: "thanos"
+  endpoint: "minio.example.com:9000"
+  access_key: "minio"
+  secret_key: "minio123"
+  insecure: true
+  signature_version2: false
+  http_config:
+    idle_conn_timeout: 90s
+    response_header_timeout: 2m

examples/objstore-s3.yaml

@@ -0,0 +1,15 @@
+type: S3
+config:
+  bucket: "my-thanos-bucket"
+  endpoint: "s3.amazonaws.com"
+  region: "us-east-1"
+  access_key: ""
+  secret_key: ""
+  insecure: false
+  signature_version2: false
+  put_user_metadata: {}
+  http_config:
+    idle_conn_timeout: 90s
+    response_header_timeout: 2m
+  trace:
+    enable: false