feat: allow input source from pipe (#4088)

* feat: allow input source from pipe

While performing a bit of experimentation,
I found it was easier to allow data to flow
from other segments of my machine via piping
it in, rather than flushing it to disk. This
can speed up the process when pre-processing
large amounts of data. This allow allowed me
to keep disk space low and just utilize a large
ram machine.

* (chore) update readme for new source pipe

* (fix) pipe -> stdin per PR

Author: strazzere

README.md

@@ -369,6 +369,12 @@ trufflehog huggingface --org <orgname> --user <username>
 trufflehog huggingface --model <model_id> --include-discussions --include-prs
 ```
 
+## 18. Scan stdin Input
+
+```bash
+aws s3 cp s3://example/gzipped/data.gz - | gunzip -c | trufflehog stdin
+```
+
 # :question: FAQ
 
 - All I see is `🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
@@ -411,6 +417,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - postman
 - jenkins
 - elasticsearch
+- stdin
 
 Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
 

main.go

@@ -252,6 +252,8 @@ var (
 	huggingfaceIncludeDiscussions = huggingfaceScan.Flag("include-discussions", "Include discussions in scan.").Bool()
 	huggingfaceIncludePrs         = huggingfaceScan.Flag("include-prs", "Include pull requests in scan.").Bool()
 
+	stdinInputScan = cli.Command("stdin", "Find credentials from stdin.")
+
 	analyzeCmd = analyzer.Command(cli)
 	usingTUI   = false
 )
@@ -942,6 +944,11 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 		if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
 		}
+	case stdinInputScan.FullCommand():
+		cfg := sources.StdinConfig{}
+		if ref, err = eng.ScanStdinInput(ctx, cfg); err != nil {
+			return scanMetrics, fmt.Errorf("failed to scan stdin input: %v", err)
+		}
 	default:
 		return scanMetrics, fmt.Errorf("invalid command: %s", cmd)
 	}

pkg/engine/stdin.go

@@ -0,0 +1,33 @@
+package engine
+
+import (
+	"runtime"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/stdin"
+)
+
+// ScanStdinInput scans input that is piped into the application
+func (e *Engine) ScanStdinInput(ctx context.Context, c sources.StdinConfig) (sources.JobProgressRef, error) {
+	connection := &sourcespb.Stdin{}
+	var conn anypb.Any
+	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+	if err != nil {
+		ctx.Logger().Error(err, "failed to marshal stdin connection")
+		return sources.JobProgressRef{}, err
+	}
+
+	sourceName := "trufflehog - stdin"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, stdin.SourceType)
+
+	stdinSource := &stdin.Source{}
+	if err := stdinSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+		return sources.JobProgressRef{}, err
+	}
+	return e.sourceManager.EnumerateAndScan(ctx, sourceName, stdinSource)
+}