Skip to content

feat: added "gs://" type (GCP: cloud storage bucket - GCS) output stream support #131

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ShubhamTiwary914 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: added "gs://" type (GCP: cloud storage bucket - GCS) output stream support #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 80 additions & 22 deletions ffmpeg.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,19 @@ import (
"os"
"strings"

"cloud.google.com/go/storage"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"google.golang.org/api/option"
)

// Input file URL (ffmpeg ``-i`` option)
// Input file URL (ffmpeg “-i“ option)
//
// Any supplied kwargs are passed to ffmpeg verbatim (e.g. ``t=20``,
// ``f='mp4'``, ``acodec='pcm'``, etc.).
// Any supplied kwargs are passed to ffmpeg verbatim (e.g. “t=20“,
// “f='mp4'“, “acodec='pcm'“, etc.).
//
// To tell ffmpeg to read from stdin, use ``pipe:`` as the filename.
// To tell ffmpeg to read from stdin, use “pipe:“ as the filename.
//
// Official documentation: `Main options <https://ffmpeg.org/ffmpeg.html#Main-options>`__
func Input(filename string, kwargs ...KwArgs) *Stream {
Expand All @@ -33,15 +35,15 @@ func Input(filename string, kwargs ...KwArgs) *Stream {
return NewInputNode("input", nil, args).Stream("", "")
}

// Add extra global command-line argument(s), e.g. ``-progress``.
// Add extra global command-line argument(s), e.g. “-progress“.
func (s *Stream) GlobalArgs(args ...string) *Stream {
if s.Type != "OutputStream" {
panic("cannot overwrite outputs on non-OutputStream")
}
return NewGlobalNode("global_args", []*Stream{s}, args, nil).Stream("", "")
}

// Overwrite output files without asking (ffmpeg ``-y`` option)
// Overwrite output files without asking (ffmpeg “-y“ option)
//
// Official documentation: `Main options <https://ffmpeg.org/ffmpeg.html#Main-options>`_
func (s *Stream) OverwriteOutput(stream *Stream) *Stream {
Expand All @@ -56,28 +58,28 @@ func MergeOutputs(streams ...*Stream) *Stream {
return NewMergeOutputsNode("merge_output", streams).Stream("", "")
}

//Output file URL
// Output file URL
//
// Syntax:
// `ffmpeg.Output([]*Stream{stream1, stream2, stream3...}, filename, kwargs)`
// Syntax:
// `ffmpeg.Output([]*Stream{stream1, stream2, stream3...}, filename, kwargs)`
//
// Any supplied keyword arguments are passed to ffmpeg verbatim (e.g.
// ``t=20``, ``f='mp4'``, ``acodec='pcm'``, ``vcodec='rawvideo'``,
// etc.). Some keyword-arguments are handled specially, as shown below.
// Any supplied keyword arguments are passed to ffmpeg verbatim (e.g.
// ``t=20``, ``f='mp4'``, ``acodec='pcm'``, ``vcodec='rawvideo'``,
// etc.). Some keyword-arguments are handled specially, as shown below.
//
// Args:
// video_bitrate: parameter for ``-b:v``, e.g. ``video_bitrate=1000``.
// audio_bitrate: parameter for ``-b:a``, e.g. ``audio_bitrate=200``.
// format: alias for ``-f`` parameter, e.g. ``format='mp4'``
// (equivalent to ``f='mp4'``).
// Args:
// video_bitrate: parameter for ``-b:v``, e.g. ``video_bitrate=1000``.
// audio_bitrate: parameter for ``-b:a``, e.g. ``audio_bitrate=200``.
// format: alias for ``-f`` parameter, e.g. ``format='mp4'``
// (equivalent to ``f='mp4'``).
//
// If multiple streams are provided, they are mapped to the same
// output.
// If multiple streams are provided, they are mapped to the same
// output.
//
// To tell ffmpeg to write to stdout, use ``pipe:`` as the filename.
// To tell ffmpeg to write to stdout, use ``pipe:`` as the filename.
//
// Official documentation: `Synopsis <https://ffmpeg.org/ffmpeg.html#Synopsis>`__
// """
// Official documentation: `Synopsis <https://ffmpeg.org/ffmpeg.html#Synopsis>`__
// """
func Output(streams []*Stream, fileName string, kwargs ...KwArgs) *Stream {
args := MergeKwArgs(kwargs)
if !args.HasKey("filename") {
Expand All @@ -96,6 +98,8 @@ func (s *Stream) Output(fileName string, kwargs ...KwArgs) *Stream {
}
if strings.HasPrefix(fileName, "s3://") {
return s.outputS3Stream(fileName, kwargs...)
} else if strings.HasPrefix(fileName, "gs://") {
return s.outputGSStream(fileName, kwargs...)
}
return Output([]*Stream{s}, fileName, kwargs...)
}
Expand Down Expand Up @@ -136,3 +140,57 @@ func (s *Stream) outputS3Stream(fileName string, kwargs ...KwArgs) *Stream {
o.Context = context.WithValue(o.Context, "run_hook", &runHook)
return o
}

func (s *Stream) outputGSStream(fileName string, kwargs ...KwArgs) *Stream {
r, w := io.Pipe()
fileL := strings.SplitN(strings.TrimPrefix(fileName, "gs://"), "/", 2)
if len(fileL) != 2 {
log.Panic("GCS file format not valid")
}
args := MergeKwArgs(kwargs)

//context from aorgs (optional)
ctx, ok := args.PopDefault("ctx", context.Background()).(context.Context)
if !ok {
ctx = context.Background()
}
//auth with GCP service account credentials file
credsFile, _ := args.PopDefault("credentials_file", "").(string)
var clientOpts []option.ClientOption
if credsFile != "" {
clientOpts = append(clientOpts, option.WithCredentialsFile(credsFile))
} else {
log.Panic("GCS Service Account Credentials file 'credentials_file' kwarg required in Output")
}

bucket, object := fileL[0], fileL[1]
o := Output([]*Stream{s}, "pipe:", args).WithOutput(w, os.Stdout)

done := make(chan struct{})
runHook := RunHook{
f: func() {
defer func() {
done <- struct{}{}
}()

client, err := storage.NewClient(ctx, clientOpts...)
if err != nil {
log.Println("failed to create GCS client:", err)
return
}
defer client.Close()

wc := client.Bucket(bucket).Object(object).NewWriter(ctx)
if _, err := io.Copy(wc, r); err != nil {
log.Println("upload fail:", err)
}
if err := wc.Close(); err != nil {
log.Println("close writer fail:", err)
}
},
done: done,
closer: w,
}
o.Context = context.WithValue(o.Context, "run_hook", &runHook)
return o
}
65 changes: 63 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,72 @@
module github.com/u2takey/ffmpeg-go

go 1.16
go 1.23.0

toolchain go1.24.3

require (
github.com/aws/aws-sdk-go v1.38.20
github.com/disintegration/imaging v1.6.2
github.com/stretchr/testify v1.5.1
github.com/stretchr/testify v1.10.0
github.com/u2takey/go-utils v0.3.1
gocv.io/x/gocv v0.25.0
)

require (
cel.dev/expr v0.20.0 // indirect
cloud.google.com/go v0.121.0 // indirect
cloud.google.com/go/auth v0.16.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.6.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/monitoring v1.24.0 // indirect
cloud.google.com/go/storage v1.54.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.14.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
go.opentelemetry.io/otel v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect
go.opentelemetry.io/otel/sdk v1.35.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
go.opentelemetry.io/otel/trace v1.35.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
golang.org/x/image v0.0.0-20220302094943-723b81ca9867 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.14.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.25.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/api v0.232.0 // indirect
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250505200425-f936aa4a68b2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect
google.golang.org/grpc v1.72.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)