Add optional opentelemetry file export tracing (#723)

Add opt-in functionality (with the `trace` feature) that writes OpenTelemetry File Export tracing information during build and compile. The tracing data includes buildpack attributes (like buildpack id, name), spans for build and detect, events for `build-success`, `detect-pass`, `detect-fail`, and also sets errors and span status in the case of errors.

Author: joshwlewis

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `libcnb`:
+  - An optional `trace` feature has been added that emits OpenTelemetry tracing
+    data to a [File Export](https://opentelemetry.io/docs/specs/otel/protocol/file-exporter/). ([#723](https://github.com/heroku/libcnb.rs/pull/723))
 
 ## [0.16.0] - 2023-11-17
 

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "test-buildpacks/readonly-layer-files",
     "test-buildpacks/sbom",
     "test-buildpacks/store",
+    "test-buildpacks/tracing",
 ]
 
 [workspace.package]

clippy.toml

@@ -1 +1,2 @@
 allow-unwrap-in-tests = true
+doc-valid-idents = ["OpenTelemetry", ".."]

libcnb/Cargo.toml

@@ -14,16 +14,23 @@ include = ["src/**/*", "LICENSE", "README.md"]
 [lints]
 workspace = true
 
+[features]
+trace = ["dep:opentelemetry", "dep:opentelemetry_sdk", "dep:opentelemetry-stdout"]
+
 [dependencies]
 anyhow = { version = "1.0.75", optional = true }
 cyclonedx-bom = { version = "0.4.3", optional = true }
 libcnb-common.workspace = true
 libcnb-data.workspace = true
 libcnb-proc-macros.workspace = true
+opentelemetry = { version = "0.21.0", optional = true }
+opentelemetry_sdk = { version = "0.21.0", optional = true }
+opentelemetry-stdout = { version = "0.2.0", optional = true, features = ["trace"] }
 serde = { version = "1.0.192", features = ["derive"] }
 thiserror = "1.0.50"
 toml.workspace = true
 
 [dev-dependencies]
 fastrand = "2.0.1"
 tempfile = "3.8.1"
+serde_json = "1.0.108"

libcnb/src/lib.rs

@@ -18,6 +18,8 @@ mod error;
 mod exit_code;
 mod platform;
 mod runtime;
+#[cfg(feature = "trace")]
+mod tracing;
 mod util;
 
 pub use buildpack::Buildpack;
@@ -27,6 +29,9 @@ pub use libcnb_common::toml_file::*;
 pub use platform::*;
 pub use runtime::*;
 
+#[cfg(all(test, not(feature = "trace")))]
+use serde_json as _;
+
 /// Provides types for CNB data formats. Is a re-export of the `libcnb-data` crate.
 #[doc(inline)]
 pub use libcnb_data as data;

libcnb/src/runtime.rs

@@ -5,9 +5,12 @@ use crate::detect::{DetectContext, InnerDetectResult};
 use crate::error::Error;
 use crate::platform::Platform;
 use crate::sbom::cnb_sbom_path;
+#[cfg(feature = "trace")]
+use crate::tracing::start_trace;
 use crate::util::is_not_found_error_kind;
 use crate::{exit_code, TomlFileError, LIBCNB_SUPPORTED_BUILDPACK_API};
 use libcnb_common::toml_file::{read_toml_file, write_toml_file};
+use libcnb_data::buildpack::ComponentBuildpackDescriptor;
 use libcnb_data::store::Store;
 use serde::de::DeserializeOwned;
 use serde::Deserialize;
@@ -121,31 +124,63 @@ pub fn libcnb_runtime_detect<B: Buildpack>(
 ) -> crate::Result<i32, B::Error> {
     let app_dir = env::current_dir().map_err(Error::CannotDetermineAppDirectory)?;
 
+    let buildpack_dir = read_buildpack_dir()?;
+
+    let buildpack_descriptor: ComponentBuildpackDescriptor<<B as Buildpack>::Metadata> =
+        read_buildpack_descriptor()?;
+
+    #[cfg(feature = "trace")]
+    let mut trace = start_trace(&buildpack_descriptor.buildpack, "detect");
+
+    let mut trace_error = |err: &dyn std::error::Error| {
+        #[cfg(feature = "trace")]
+        trace.set_error(err);
+    };
     let stack_id: StackId = env::var("CNB_STACK_ID")
         .map_err(Error::CannotDetermineStackId)
-        .and_then(|stack_id_string| stack_id_string.parse().map_err(Error::StackIdError))?;
-
-    let platform = B::Platform::from_path(&args.platform_dir_path)
-        .map_err(Error::CannotCreatePlatformFromPath)?;
+        .and_then(|stack_id_string| stack_id_string.parse().map_err(Error::StackIdError))
+        .map_err(|err| {
+            trace_error(&err);
+            err
+        })?;
+
+    let platform = B::Platform::from_path(&args.platform_dir_path).map_err(|inner_err| {
+        let err = Error::CannotCreatePlatformFromPath(inner_err);
+        trace_error(&err);
+        err
+    })?;
 
     let build_plan_path = args.build_plan_path;
 
     let detect_context = DetectContext {
         app_dir,
+        buildpack_dir,
         stack_id,
         platform,
-        buildpack_dir: read_buildpack_dir()?,
-        buildpack_descriptor: read_buildpack_descriptor()?,
+        buildpack_descriptor,
     };
 
-    match buildpack.detect(detect_context)?.0 {
-        InnerDetectResult::Fail => Ok(exit_code::DETECT_DETECTION_FAILED),
+    let detect_result = buildpack.detect(detect_context).map_err(|err| {
+        trace_error(&err);
+        err
+    })?;
+
+    match detect_result.0 {
+        InnerDetectResult::Fail => {
+            #[cfg(feature = "trace")]
+            trace.add_event("detect-failed");
+            Ok(exit_code::DETECT_DETECTION_FAILED)
+        }
         InnerDetectResult::Pass { build_plan } => {
             if let Some(build_plan) = build_plan {
-                write_toml_file(&build_plan, build_plan_path)
-                    .map_err(Error::CannotWriteBuildPlan)?;
+                write_toml_file(&build_plan, build_plan_path).map_err(|inner_err| {
+                    let err = Error::CannotWriteBuildPlan(inner_err);
+                    trace_error(&err);
+                    err
+                })?;
             }
-
+            #[cfg(feature = "trace")]
+            trace.add_event("detect-passed");
             Ok(exit_code::DETECT_DETECTION_PASSED)
         }
     }
@@ -163,31 +198,63 @@ pub fn libcnb_runtime_build<B: Buildpack>(
 
     let app_dir = env::current_dir().map_err(Error::CannotDetermineAppDirectory)?;
 
+    let buildpack_dir = read_buildpack_dir()?;
+
+    let buildpack_descriptor: ComponentBuildpackDescriptor<<B as Buildpack>::Metadata> =
+        read_buildpack_descriptor()?;
+
+    #[cfg(feature = "trace")]
+    let mut trace = start_trace(&buildpack_descriptor.buildpack, "build");
+
+    let mut trace_error = |err: &dyn std::error::Error| {
+        #[cfg(feature = "trace")]
+        trace.set_error(err);
+    };
+
     let stack_id: StackId = env::var("CNB_STACK_ID")
         .map_err(Error::CannotDetermineStackId)
-        .and_then(|stack_id_string| stack_id_string.parse().map_err(Error::StackIdError))?;
-
-    let platform = Platform::from_path(&args.platform_dir_path)
-        .map_err(Error::CannotCreatePlatformFromPath)?;
+        .and_then(|stack_id_string| stack_id_string.parse().map_err(Error::StackIdError))
+        .map_err(|err| {
+            trace_error(&err);
+            err
+        })?;
+
+    let platform = Platform::from_path(&args.platform_dir_path).map_err(|inner_err| {
+        let err = Error::CannotCreatePlatformFromPath(inner_err);
+        trace_error(&err);
+        err
+    })?;
 
-    let buildpack_plan =
-        read_toml_file(&args.buildpack_plan_path).map_err(Error::CannotReadBuildpackPlan)?;
+    let buildpack_plan = read_toml_file(&args.buildpack_plan_path).map_err(|inner_err| {
+        let err = Error::CannotReadBuildpackPlan(inner_err);
+        trace_error(&err);
+        err
+    })?;
 
     let store = match read_toml_file::<Store>(layers_dir.join("store.toml")) {
         Err(TomlFileError::IoError(io_error)) if is_not_found_error_kind(&io_error) => Ok(None),
         other => other.map(Some),
     }
-    .map_err(Error::CannotReadStore)?;
+    .map_err(Error::CannotReadStore)
+    .map_err(|err| {
+        trace_error(&err);
+        err
+    })?;
 
-    let build_result = buildpack.build(BuildContext {
+    let build_context = BuildContext {
         layers_dir: layers_dir.clone(),
         app_dir,
         stack_id,
         platform,
         buildpack_plan,
-        buildpack_dir: read_buildpack_dir()?,
-        buildpack_descriptor: read_buildpack_descriptor()?,
+        buildpack_dir,
+        buildpack_descriptor,
         store,
+    };
+
+    let build_result = buildpack.build(build_context).map_err(|err| {
+        trace_error(&err);
+        err
     })?;
 
     match build_result.0 {
@@ -198,31 +265,47 @@ pub fn libcnb_runtime_build<B: Buildpack>(
             launch_sboms,
         } => {
             if let Some(launch) = launch {
-                write_toml_file(&launch, layers_dir.join("launch.toml"))
-                    .map_err(Error::CannotWriteLaunch)?;
+                write_toml_file(&launch, layers_dir.join("launch.toml")).map_err(|inner_err| {
+                    let err = Error::CannotWriteLaunch(inner_err);
+                    trace_error(&err);
+                    err
+                })?;
             };
 
             if let Some(store) = store {
-                write_toml_file(&store, layers_dir.join("store.toml"))
-                    .map_err(Error::CannotWriteStore)?;
+                write_toml_file(&store, layers_dir.join("store.toml")).map_err(|inner_err| {
+                    let err = Error::CannotWriteStore(inner_err);
+                    trace_error(&err);
+                    err
+                })?;
             };
 
             for build_sbom in build_sboms {
                 fs::write(
                     cnb_sbom_path(&build_sbom.format, &layers_dir, "build"),
                     &build_sbom.data,
                 )
-                .map_err(Error::CannotWriteBuildSbom)?;
+                .map_err(Error::CannotWriteBuildSbom)
+                .map_err(|err| {
+                    trace_error(&err);
+                    err
+                })?;
             }
 
             for launch_sbom in launch_sboms {
                 fs::write(
                     cnb_sbom_path(&launch_sbom.format, &layers_dir, "launch"),
                     &launch_sbom.data,
                 )
-                .map_err(Error::CannotWriteLaunchSbom)?;
+                .map_err(Error::CannotWriteLaunchSbom)
+                .map_err(|err| {
+                    trace_error(&err);
+                    err
+                })?;
             }
 
+            #[cfg(feature = "trace")]
+            trace.add_event("build-success");
             Ok(exit_code::GENERIC_SUCCESS)
         }
     }