feat(o11y): self-logs via shared entry-point zap->OTel bridge + OTLP log export (PIPE-1066)

cmd/blitz/main.go

@@ -24,6 +24,7 @@ import (
 	"github.com/observiq/blitz/internal/dispatch"
 	"github.com/observiq/blitz/internal/logging"
 	"github.com/observiq/blitz/internal/service"
+	"github.com/observiq/blitz/internal/telemetry/logs"
 	"github.com/observiq/blitz/internal/telemetry/metrics"
 	"github.com/observiq/blitz/internal/telemetry/traces"
 	"github.com/observiq/blitz/output"
@@ -120,16 +121,42 @@ func run(cmd *cobra.Command, args []string) error {
 	}
 	defer func() { _ = logger.Sync() }()
 
+	// Create signal context for graceful shutdown.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Build blitz's self-telemetry bundle. The log provider is constructed and
+	// the logger bridged FIRST, before any startup logging, so every line from
+	// here on is exported when log export is enabled. blitz bridges the logger
+	// once, here at the process entry point, and shares the bridged logger with
+	// every component; components do not re-bridge, since a bridged zap logger
+	// propagates to the child loggers they derive. Metrics leave the provider
+	// nil so they fall back to the process-global Prometheus provider that
+	// setupMetrics installs.
+	tel := embed.TelemetrySettings{PerBatchSpans: cfg.Telemetry.Traces.PerBatchSpans}
+	logExportEnabled := cfg.Telemetry.Logs.OTLPEndpoint != ""
+	if logExportEnabled {
+		otlpLogs, lerr := logs.NewOTLP(ctx, cfg.Telemetry.Logs.OTLPEndpoint, cfg.Telemetry.Logs.Insecure)
+		if lerr != nil {
+			logger.Error("Failed to enable self-telemetry log export", zap.Error(lerr))
+			return lerr
+		}
+		defer func() { _ = otlpLogs.Shutdown(context.Background()) }()
+		tel.LoggerProvider = otlpLogs.Provider()
+	}
+	logger = tel.BridgedLogger(logger)
+	tel.Logger = logger
+
 	logger.Info("blitz started")
+	if logExportEnabled {
+		logger.Info("self-telemetry log export enabled",
+			zap.String("endpoint", cfg.Telemetry.Logs.OTLPEndpoint))
+	}
 
 	// Emit Warn-level banners for any deprecated generator types
 	// configured by the user. Fires once per startup, not per record.
 	config.LogGeneratorDeprecations(logger, cfg)
 
-	// Create signal context for graceful shutdown
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	if err := setupMetrics(ctx, cfg, logger); err != nil {
 		logger.Error("Failed to setup metrics", zap.Error(err))
 		return err
@@ -144,16 +171,9 @@ func run(cmd *cobra.Command, args []string) error {
 		cancel()
 	}()
 
-	// Blitz routes its own self-telemetry through this bundle. Metrics leave
-	// the provider nil so they fall back to the process-global provider
-	// configured by setupMetrics (Prometheus). Trace export is opt-in via the
-	// telemetry.traces config: when an OTLP endpoint is set, spans export
-	// there; otherwise the nil TracerProvider means spans are created but
-	// dropped by the global no-op provider.
-	tel := embed.TelemetrySettings{
-		Logger:        logger,
-		PerBatchSpans: cfg.Telemetry.Traces.PerBatchSpans,
-	}
+	// Trace export is opt-in via the telemetry.traces config: when an OTLP
+	// endpoint is set, spans export there; otherwise the nil TracerProvider
+	// means spans are created but dropped by the global no-op provider.
 	if cfg.Telemetry.Traces.OTLPEndpoint != "" {
 		otlpTraces, terr := traces.NewOTLP(ctx, cfg.Telemetry.Traces.OTLPEndpoint, cfg.Telemetry.Traces.Insecure)
 		if terr != nil {

config/loader.go

@@ -149,6 +149,11 @@ func LoadModules(yamlBytes []byte, opts EmbedOpts) ([]embed.ProducerModule, erro
 	if logger == nil {
 		logger = zap.NewNop()
 	}
+	// Bridge the logger once here, at the embed construction entry point, and
+	// share the bridged logger with every generator. Components do not
+	// re-bridge: a bridged zap logger propagates to the child loggers they
+	// derive. A nil LoggerProvider leaves the logger zap-only.
+	logger = opts.Telemetry.BridgedLogger(logger)
 	cfg, err := Load(yamlBytes, LoadOpts{EnvOverrides: opts.EnvOverrides})
 	if err != nil {
 		return nil, err

docs/embed.md

@@ -81,38 +81,55 @@ import (
 func main() {
     logger := zap.NewNop()
 
-    // 1. Host owns the consumers and ambient resources.
+    // 1. Build ONE self-telemetry bundle. Every field is optional and nil-safe.
+    //    blitz builds its own metrics, spans, and log bridge from these. See
+    //    "Self-telemetry" below.
+    tel := embed.TelemetrySettings{
+        Logger:         logger,           // blitz's internal zap logger (nil -> nop)
+        MeterProvider:  myMeterProvider,  // nil -> process-global provider
+        TracerProvider: myTracerProvider, // nil -> process-global provider
+        LoggerProvider: myLoggerProvider, // nil -> logs stay zap-only (no bridge)
+    }
+
+    // 2. Host owns the consumers, ambient resources, and the SAME bundle.
     host := embed.Host{
-        Logs:    myLogConsumer,
-        Metrics: myMetricConsumer, // optional, required when a metric generator is wired
-        Traces:  myTraceConsumer,  // optional, required when a trace generator is wired
-        Logger:  logger,
+        Logs:      myLogConsumer,
+        Metrics:   myMetricConsumer, // optional, required when a metric generator is wired
+        Traces:    myTraceConsumer,  // optional, required when a trace generator is wired
+        Telemetry: tel,
         // Resource also available.
     }
 
-    // 2. Construct modules, passing the appropriate consumer from host.
-    apacheGen, _ := apache.New(logger, /*workers*/ 1, /*rate*/ time.Second, host.Logs)
+    // 3. Bridge the logger once, then construct modules. Components do NOT
+    //    re-bridge: a bridged zap logger propagates to the child loggers they
+    //    derive, so one bridge here covers them all. config.LoadModules does
+    //    this for you on the YAML path. Each component builds its metrics and
+    //    tracer from tel.
+    logger = tel.BridgedLogger(logger)
+    apacheGen, _ := apache.New(logger, /*workers*/ 1, /*rate*/ time.Second, host.Logs, tel)
     hmGen, _ := hostmetrics.New(hostmetrics.Config{
-        Logger:   logger,
-        Workers:  1,
-        Rate:     10 * time.Second,
-        OS:       "linux",
-        Consumer: host.Metrics,
+        Logger:    logger,
+        Workers:   1,
+        Rate:      10 * time.Second,
+        OS:        "linux",
+        Consumer:  host.Metrics,
+        Telemetry: tel,
     })
     tracesGen, _ := traces.New(traces.Config{
-        Logger:   logger,
-        Workers:  1,
-        Rate:     time.Second,
-        Consumer: host.Traces,
+        Logger:    logger,
+        Workers:   1,
+        Rate:      time.Second,
+        Consumer:  host.Traces,
+        Telemetry: tel,
     })
 
-    // 3. Build the runner.
+    // 4. Build the runner.
     runner, err := embed.New(embed.Config{
         Modules: []embed.ProducerModule{apacheGen, hmGen, tracesGen},
     })
     if err != nil { /* ... */ }
 
-    // 4. Start, run, stop.
+    // 5. Start, run, stop.
     ctx := context.Background()
     if err := runner.Start(ctx, host); err != nil { /* ... */ }
 
@@ -152,6 +169,32 @@ embed.Config{
 
 Consumer errors are best-effort: blitz logs the error, increments a `consumer_errors` counter, and continues producing. A consumer that wants stricter semantics can return errors and observe them on the metric.
 
+## Self-telemetry (logs, metrics, traces)
+
+Blitz emits its **own** operational telemetry (distinct from the data it generates), and an embedding host can route all three signals through host-supplied OTel providers. This is separate from the consumers above: consumers receive the generated data; the providers below receive blitz's internal observability.
+
+One `embed.TelemetrySettings` bundle carries everything:
+
+```go
+type TelemetrySettings struct {
+    Logger         *zap.Logger          // blitz's internal diagnostic logger
+    MeterProvider  metric.MeterProvider // source of blitz's metric instruments
+    TracerProvider trace.TracerProvider // source of blitz's spans
+    LoggerProvider log.LoggerProvider   // receives blitz's logs as OTel records
+    PerBatchSpans  bool                 // opt-in higher-volume per-emit-cycle spans
+}
+```
+
+**One bundle, three signals.** The host builds a single bundle and supplies the *same value* in two places: as `embed.Host.Telemetry` (used by the runner for the session-level runtime) and at generator construction (the `Telemetry` field on config-struct constructors, or the trailing `tel` argument on positional ones, and `EmbedOpts.Telemetry` when using `config.LoadModules`). Metrics and traces are built per component from that bundle at construction; the logger is bridged once and shared:
+
+- **Metrics**: `output.NewMetrics(tel.MeterProvider)` / `generator.NewMetrics(...)`, per component.
+- **Traces**: `tel.Tracer(scope)`, per component (plus the runtime's session spans).
+- **Logs**: `tel.BridgedLogger(logger)` tees zap logging into `tel.LoggerProvider` as OTel records, done **once** at the entry point (`main` for standalone; `config.LoadModules` and the runner for embed) and shared. Components receive the already-bridged logger and do not re-bridge, since a bridged zap logger propagates to the child loggers they derive. A caller constructing a generator directly, bypassing `config.LoadModules`, bridges the logger itself. This is the OTel-idiomatic path: zap stays the logging API, the OTel Logs SDK is the bridge backend.
+
+**Nil-safe fallbacks.** Every field is optional. A nil `MeterProvider` or `TracerProvider` falls back to the process-global provider; a nil `LoggerProvider` leaves logs zap-only (no bridge). A zero-value bundle therefore behaves exactly as blitz did before providers were injectable, and `embed.NopTelemetry()` returns an all-no-op bundle for hosts that route nothing.
+
+**What blitz emits:** a `blitz.session` root span covering Start to Stop with a `blitz.generator.run` child span per module; per-generator and per-output metric instruments (`blitz.generator.entries`, and the rest); and blitz's internal zap logs, bridged to OTel when a `LoggerProvider` is set.
+
 ## Resource attributes
 
 Every blitz record carries a per-record `Metadata.Resource` map describing the entity that emitted it (host, module, format, version). The three signal types are structured identically:
@@ -233,6 +276,7 @@ Two supported paths:
       Logger:        logger,
       LogConsumer:   myLogConsumer,
       TraceConsumer: myTraceConsumer,
+      Telemetry:     tel, // same bundle as embed.Host.Telemetry; see "Self-telemetry"
       // MetricConsumer also available.
       // FileGenLibrary: embeddedlibrary.FS(), // optional; nil = ./data_library/ on disk
   })

embed/host.go

@@ -1,11 +1,5 @@
 package embed
 
-import (
-	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/trace"
-	"go.uber.org/zap"
-)
-
 // Host is the bundle of consumers and ambient resources a host process
 // supplies to an embedded blitz runner.
 //
@@ -22,10 +16,6 @@ type Host struct {
 	// Traces is the destination for spans. Nil means spans are dropped.
 	Traces TraceConsumer
 
-	// Logger is the zap logger blitz uses for internal diagnostics. Nil
-	// means blitz constructs a no-op logger.
-	Logger *zap.Logger
-
 	// Resource is the per-session base resource attributes blitz applies
 	// to every emitted record before module-level overrides merge on top.
 	//
@@ -36,20 +26,17 @@ type Host struct {
 	// See cloneResource in this package.
 	Resource map[string]string
 
-	// MeterProvider is the OTel MeterProvider blitz routes its own internal
-	// metrics through (generator and output self-telemetry). Nil falls back
-	// to the process-global provider, matching standalone behavior.
-	MeterProvider metric.MeterProvider
-
-	// TracerProvider is the OTel TracerProvider blitz routes its own internal
-	// spans through. Nil falls back to the process-global provider. Reserved
-	// for the self-tracing phase; blitz emits no internal spans yet.
-	TracerProvider trace.TracerProvider
-
-	// PerBatchSpans enables the higher-volume per-emit-cycle spans once
-	// self-tracing lands. Off by default; the coarse spans do not depend on
-	// it.
-	PerBatchSpans bool
+	// Telemetry carries the OTel providers and logger blitz routes its own
+	// self-telemetry through: the zap Logger for internal diagnostics, and the
+	// Meter, Tracer, and Logger providers blitz builds its metrics, spans, and
+	// log bridge from. It is the single source for all three signals.
+	//
+	// The host builds one bundle and supplies the same value here (used by the
+	// runner for the session-level runtime) and as EmbedOpts.Telemetry (used at
+	// generator construction). Every field is optional with a nil-safe
+	// fallback, so a zero-value bundle behaves as blitz did before providers
+	// were injectable. See TelemetrySettings.
+	Telemetry TelemetrySettings
 }
 
 // cloneResource returns a defensive copy of m. Runner.Start uses it so