From 62388bdbb0db7d52bbfc8a65730e09bd0c801503 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 May 2026 13:26:34 +0800 Subject: [PATCH 1/8] feat(core): add capability override layer --- core/Cargo.lock | 5 +- core/core/src/layers/capability_override.rs | 110 +++++++++++++ core/core/src/layers/mod.rs | 3 + core/core/src/types/capability.rs | 6 +- core/testkit/Cargo.toml | 1 + core/testkit/src/utils.rs | 161 +++++++++++++++++++- core/tests/behavior/README.md | 10 ++ core/tests/behavior/async_delete.rs | 6 +- 8 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 core/core/src/layers/capability_override.rs diff --git a/core/Cargo.lock b/core/Cargo.lock index a065f0312218..fd9e54078a90 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -3307,7 +3307,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0f3f231ff5b8465063b2bf8af3b975222b4fa3cc8c07b43d01fe8e0ae15c9dd" dependencies = [ - "bindgen 0.72.1", + "bindgen", "libc", ] @@ -5224,7 +5224,7 @@ version = "0.17.3+10.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" dependencies = [ - "bindgen 0.72.1", + "bindgen", "bzip2-sys", "cc", "libc", @@ -7560,6 +7560,7 @@ dependencies = [ "opendal-layer-retry", "opendal-layer-timeout", "rand 0.10.1", + "serde_json", "sha2 0.11.0", "tokio", "uuid", diff --git a/core/core/src/layers/capability_override.rs b/core/core/src/layers/capability_override.rs new file mode 100644 index 000000000000..65cb2e0a1a37 --- /dev/null +++ b/core/core/src/layers/capability_override.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; +use std::sync::Arc; + +use crate::raw::*; +use crate::*; + +/// Layer for overriding an accessor's full capability. +/// +/// This layer updates [`Capability`] exposed by +/// [`OperatorInfo::full_capability`][crate::OperatorInfo::full_capability] +/// without changing the accessor's native capability. It is useful when the +/// backend implementation supports a capability, but the specific endpoint or +/// test setup needs to disable or tune it. +/// +/// # Examples +/// +/// ```no_run +/// use opendal_core::layers::CapabilityOverrideLayer; +/// use opendal_core::services; +/// use opendal_core::Operator; +/// use opendal_core::Result; +/// +/// # fn main() -> Result<()> { +/// let op = Operator::new(services::Memory::default())? +/// .layer(CapabilityOverrideLayer::new(|mut cap| { +/// cap.write_with_if_match = false; +/// cap.delete_max_size = Some(700); +/// cap +/// })) +/// .finish(); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone)] +pub struct CapabilityOverrideLayer { + apply: Arc Capability + Send + Sync>, +} + +impl fmt::Debug for CapabilityOverrideLayer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CapabilityOverrideLayer") + .finish_non_exhaustive() + } +} + +impl CapabilityOverrideLayer { + /// Create a new [`CapabilityOverrideLayer`]. + pub fn new(apply: impl Fn(Capability) -> Capability + Send + Sync + 'static) -> Self { + Self { + apply: Arc::new(apply), + } + } +} + +impl Layer for CapabilityOverrideLayer { + type LayeredAccess = A; + + fn layer(&self, inner: A) -> Self::LayeredAccess { + let info = inner.info(); + let apply = self.apply.clone(); + info.update_full_capability(|cap| apply(cap)); + inner + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Operator; + use crate::services; + + #[test] + fn capability_override_updates_full_capability_only() -> Result<()> { + let op = Operator::new(services::Memory::default())? + .layer(CapabilityOverrideLayer::new(|mut cap| { + cap.read = false; + cap.delete_max_size = Some(7); + cap + })) + .finish(); + + assert!(!op.info().full_capability().read); + assert_eq!(op.info().full_capability().delete_max_size, Some(7)); + + assert!(op.info().native_capability().read); + assert_ne!( + op.info().native_capability().delete_max_size, + op.info().full_capability().delete_max_size + ); + + Ok(()) + } +} diff --git a/core/core/src/layers/mod.rs b/core/core/src/layers/mod.rs index 3a1f45bf7654..f81b8c837cdc 100644 --- a/core/core/src/layers/mod.rs +++ b/core/core/src/layers/mod.rs @@ -29,6 +29,9 @@ pub(crate) use complete::CompleteLayer; mod simulate; pub use simulate::SimulateLayer; +mod capability_override; +pub use capability_override::CapabilityOverrideLayer; + mod correctness_check; pub(crate) use correctness_check::CorrectnessCheckLayer; diff --git a/core/core/src/types/capability.rs b/core/core/src/types/capability.rs index 7c49eee181b6..e883d316b5ee 100644 --- a/core/core/src/types/capability.rs +++ b/core/core/src/types/capability.rs @@ -17,6 +17,9 @@ use std::fmt::Debug; +use serde::Deserialize; +use serde::Serialize; + /// Capability defines the supported operations and their constraints for a storage Operator. /// /// # Overview @@ -62,7 +65,8 @@ use std::fmt::Debug; /// - Metadata Results: Returning metadata capabilities (e.g., `stat_has_content_length`) /// /// All capability fields are public and can be accessed directly. -#[derive(Copy, Clone, Default)] +#[derive(Copy, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Capability { /// Indicates if the operator supports metadata retrieval operations. pub stat: bool, diff --git a/core/testkit/Cargo.toml b/core/testkit/Cargo.toml index 537964a84633..a6c2396d70c4 100644 --- a/core/testkit/Cargo.toml +++ b/core/testkit/Cargo.toml @@ -38,6 +38,7 @@ opendal-layer-logging = { path = "../layers/logging", version = "0.56.0", defaul opendal-layer-retry = { path = "../layers/retry", version = "0.56.0", default-features = false } opendal-layer-timeout = { path = "../layers/timeout", version = "0.56.0", default-features = false } rand = { workspace = true } +serde_json = { workspace = true } sha2 = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"] } uuid = { workspace = true } diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs index e356ba038046..01c219ebed6a 100644 --- a/core/testkit/src/utils.rs +++ b/core/testkit/src/utils.rs @@ -19,14 +19,23 @@ use std::collections::HashMap; use std::env; use std::sync::LazyLock; +use opendal_core::Capability; +use opendal_core::Error; +use opendal_core::ErrorKind; use opendal_core::Operator; use opendal_core::Result; +use opendal_core::layers::CapabilityOverrideLayer; use opendal_layer_logging::LoggingLayer; use opendal_layer_retry::RetryLayer; use opendal_layer_timeout::TimeoutLayer; +use serde_json::Map; +use serde_json::Number; +use serde_json::Value; use sha2::Digest; use sha2::Sha256; +const OPENDAL_TEST_CAPABILITY_OVERRIDES: &str = "OPENDAL_TEST_CAPABILITY_OVERRIDES"; + pub(crate) fn sha256_digest(data: impl AsRef<[u8]>) -> String { use std::fmt::Write; @@ -86,7 +95,14 @@ pub fn init_test_service() -> Result> { // string-based scheme uses a hyphen ('-') as the connector let scheme = scheme.replace('_', "-"); - let op = Operator::via_iter(scheme, cfg).expect("must succeed"); + let mut op = Operator::via_iter(scheme, cfg).expect("must succeed"); + + if let Ok(overrides) = env::var(OPENDAL_TEST_CAPABILITY_OVERRIDES) { + let overrides = CapabilityOverrides::parse(&overrides)?; + op = op.layer(CapabilityOverrideLayer::new(move |cap| { + overrides.apply(cap) + })); + } let op = op .layer(LoggingLayer::default()) @@ -95,3 +111,146 @@ pub fn init_test_service() -> Result> { Ok(Some(op)) } + +#[derive(Clone, Debug, Default)] +struct CapabilityOverrides { + values: Map, +} + +impl CapabilityOverrides { + fn parse(input: &str) -> Result { + let mut overrides = Self::default(); + + for token in input.split(',').map(str::trim).filter(|v| !v.is_empty()) { + let (name, value) = parse_capability_override(token)?; + overrides.values.insert(name.to_string(), value); + overrides.try_apply(Capability::default()).map_err(|err| { + invalid_capability_override(token, &format!("failed to apply override: {err}")) + })?; + } + + Ok(overrides) + } + + fn apply(&self, cap: Capability) -> Capability { + self.try_apply(cap) + .expect("capability overrides must be validated before applying") + } + + fn try_apply(&self, cap: Capability) -> Result { + let mut value = serde_json::to_value(cap).map_err(|err| { + Error::new( + ErrorKind::Unexpected, + format!("failed to serialize capability: {err}"), + ) + })?; + let object = value.as_object_mut().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "serialized capability must be a JSON object", + ) + })?; + object.extend(self.values.clone()); + + serde_json::from_value(value).map_err(|err| { + Error::new( + ErrorKind::ConfigInvalid, + format!("failed to deserialize capability overrides: {err}"), + ) + }) + } +} + +fn parse_capability_override(token: &str) -> Result<(&str, Value)> { + if let Some(name) = token.strip_prefix('+') { + return Ok((name.trim(), Value::Bool(true))); + } + + if let Some(name) = token.strip_prefix('-') { + return Ok((name.trim(), Value::Bool(false))); + } + + let Some((name, value)) = token.split_once('=') else { + return Err(invalid_capability_override( + token, + "expected `+capability`, `-capability`, or `capability=value`", + )); + }; + + Ok(( + name.trim(), + parse_capability_value(value.trim()) + .map_err(|err| invalid_capability_override(token, &err.to_string()))?, + )) +} + +fn invalid_capability_override(token: &str, reason: &str) -> Error { + Error::new( + ErrorKind::ConfigInvalid, + format!("invalid {OPENDAL_TEST_CAPABILITY_OVERRIDES} entry `{token}`: {reason}"), + ) +} + +fn parse_capability_value(value: &str) -> Result { + match value { + "true" | "on" | "yes" => Ok(Value::Bool(true)), + "false" | "off" | "no" => Ok(Value::Bool(false)), + "none" | "null" | "unset" => Ok(Value::Null), + _ => value + .parse::() + .map(|v| Value::Number(Number::from(v))) + .map_err(|_| { + Error::new( + ErrorKind::ConfigInvalid, + "expected a boolean, non-negative integer, or `none`", + ) + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_capability_overrides() { + let overrides = CapabilityOverrides::parse("-read,+write_can_append,delete_max_size=7") + .expect("override must parse"); + let cap = overrides.apply(Capability { + read: true, + delete_max_size: Some(1000), + ..Default::default() + }); + + assert!(!cap.read); + assert!(cap.write_can_append); + assert_eq!(cap.delete_max_size, Some(7)); + } + + #[test] + fn parse_bool_assignments_and_unset_sizes() { + let overrides = CapabilityOverrides::parse("read=false,write=true,delete_max_size=none") + .expect("override must parse"); + let cap = overrides.apply(Capability { + read: true, + delete_max_size: Some(1000), + ..Default::default() + }); + + assert!(!cap.read); + assert!(cap.write); + assert_eq!(cap.delete_max_size, None); + } + + #[test] + fn reject_unknown_capability() { + let err = CapabilityOverrides::parse("-not_a_capability").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reject_invalid_capability_type() { + let err = CapabilityOverrides::parse("read=1").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } +} diff --git a/core/tests/behavior/README.md b/core/tests/behavior/README.md index 186c6c030801..84bf714545f1 100644 --- a/core/tests/behavior/README.md +++ b/core/tests/behavior/README.md @@ -32,6 +32,16 @@ OPENDAL_FS_ROOT=/tmp/ Notice: If the env variables are not set, all behavior tests will be skipped by default. +### Override Capability + +Behavior tests are selected by the operator's full capability. Test setups can override capability with `OPENDAL_TEST_CAPABILITY_OVERRIDES`: + +```shell +OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,delete_max_size=700 +``` + +Use `-capability` to disable a boolean capability, `+capability` to enable one, and `capability=value` to set boolean or numeric capability values. + ## Run Use `OPENDAL_TEST` to control which service to test: diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs index 90b5bdecf19a..3d1893a7cd46 100644 --- a/core/tests/behavior/async_delete.rs +++ b/core/tests/behavior/async_delete.rs @@ -17,7 +17,7 @@ use anyhow::Result; use futures::TryStreamExt; -use opendal::raw::Access; +use opendal::layers::CapabilityOverrideLayer; use opendal::raw::OpDelete; use crate::*; @@ -351,7 +351,7 @@ pub async fn test_batch_delete(op: Operator) -> Result<()> { } cap.delete_max_size = Some(2); - op.inner().info().update_full_capability(|_| cap); + let op = op.layer(CapabilityOverrideLayer::new(move |_| cap)); let mut files = Vec::new(); for _ in 0..5 { @@ -385,7 +385,7 @@ pub async fn test_batch_delete_with_version(op: Operator) -> Result<()> { } cap.delete_max_size = Some(2); - op.inner().info().update_full_capability(|_| cap); + let op = op.layer(CapabilityOverrideLayer::new(move |_| cap)); let mut files = Vec::new(); for _ in 0..5 { From 996ce9b8a2139ddf8b30f2a3033bd5a912490a13 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 May 2026 13:32:13 +0800 Subject: [PATCH 2/8] refactor(services/s3): use capability overrides in tests --- .github/services/s3/0_minio_s3/action.yml | 1 + .github/services/s3/aws_s3/action.yml | 5 ++ .../s3/aws_s3_with_list_objects_v1/action.yml | 1 + .../services/s3/aws_s3_with_sse_c/action.yml | 1 + .../s3/aws_s3_with_versioning/action.yml | 4 +- .../s3/aws_s3_with_virtual_host/action.yml | 1 + .../disable_action.yml | 2 +- .../s3/ceph_rados_s3/disable_action.yml | 1 + .../s3/minio_s3_with_anonymous/action.yml | 1 + .../minio_s3_with_list_objects_v1/action.yml | 1 + .../s3/minio_s3_with_versioning/action.yml | 2 +- .github/services/s3/r2/disabled_action.yml | 1 + core/services/s3/src/backend.rs | 58 ++++++++++++++----- core/services/s3/src/config.rs | 14 ++++- core/services/s3/src/docs.md | 2 + 15 files changed, 76 insertions(+), 19 deletions(-) diff --git a/.github/services/s3/0_minio_s3/action.yml b/.github/services/s3/0_minio_s3/action.yml index 2ee8bc8bb314..3ae887e301cf 100644 --- a/.github/services/s3/0_minio_s3/action.yml +++ b/.github/services/s3/0_minio_s3/action.yml @@ -43,4 +43,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append EOF diff --git a/.github/services/s3/aws_s3/action.yml b/.github/services/s3/aws_s3/action.yml index 13622c405911..974a4b57d90a 100644 --- a/.github/services/s3/aws_s3/action.yml +++ b/.github/services/s3/aws_s3/action.yml @@ -32,3 +32,8 @@ runs: OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key OPENDAL_S3_REGION: op://services/s3/region + + - name: Add capability overrides + shell: bash + run: | + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_list_objects_v1/action.yml b/.github/services/s3/aws_s3_with_list_objects_v1/action.yml index 90bbb961f105..feb14ab57ad8 100644 --- a/.github/services/s3/aws_s3_with_list_objects_v1/action.yml +++ b/.github/services/s3/aws_s3_with_list_objects_v1/action.yml @@ -37,3 +37,4 @@ runs: shell: bash run: | echo "OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml index 050a17162adf..bdb11c817ca0 100644 --- a/.github/services/s3/aws_s3_with_sse_c/action.yml +++ b/.github/services/s3/aws_s3_with_sse_c/action.yml @@ -40,4 +40,5 @@ runs: OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256 OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA= OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw== + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append EOF diff --git a/.github/services/s3/aws_s3_with_versioning/action.yml b/.github/services/s3/aws_s3_with_versioning/action.yml index 01120c5a1bb6..8a36e05cdd0c 100644 --- a/.github/services/s3/aws_s3_with_versioning/action.yml +++ b/.github/services/s3/aws_s3_with_versioning/action.yml @@ -33,7 +33,7 @@ runs: OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key OPENDAL_S3_REGION: op://services/s3/region - - name: Add extra settings + - name: Add capability overrides shell: bash run: | - echo "OPENDAL_S3_ENABLE_VERSIONING=true" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_can_append" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_virtual_host/action.yml b/.github/services/s3/aws_s3_with_virtual_host/action.yml index 08056a146b08..9ee8ec3eba44 100644 --- a/.github/services/s3/aws_s3_with_virtual_host/action.yml +++ b/.github/services/s3/aws_s3_with_virtual_host/action.yml @@ -37,3 +37,4 @@ runs: shell: bash run: | echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append" >> $GITHUB_ENV diff --git a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml index 71b550d91e25..4aa37ec559bf 100644 --- a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml +++ b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml @@ -43,6 +43,6 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_ENABLE_VERSIONING=true OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on + OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_can_append EOF diff --git a/.github/services/s3/ceph_rados_s3/disable_action.yml b/.github/services/s3/ceph_rados_s3/disable_action.yml index 54e6e7049ddb..2d3b51dca6bc 100644 --- a/.github/services/s3/ceph_rados_s3/disable_action.yml +++ b/.github/services/s3/ceph_rados_s3/disable_action.yml @@ -42,4 +42,5 @@ runs: OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append EOF diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml index 3c529f06eeca..70aeea083f6f 100644 --- a/.github/services/s3/minio_s3_with_anonymous/action.yml +++ b/.github/services/s3/minio_s3_with_anonymous/action.yml @@ -49,4 +49,5 @@ runs: OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_ALLOW_ANONYMOUS=on OPENDAL_S3_DISABLE_EC2_METADATA=on + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append EOF diff --git a/.github/services/s3/minio_s3_with_list_objects_v1/action.yml b/.github/services/s3/minio_s3_with_list_objects_v1/action.yml index 46904d7aaf92..98dc9ae406f8 100644 --- a/.github/services/s3/minio_s3_with_list_objects_v1/action.yml +++ b/.github/services/s3/minio_s3_with_list_objects_v1/action.yml @@ -44,4 +44,5 @@ runs: OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append EOF diff --git a/.github/services/s3/minio_s3_with_versioning/action.yml b/.github/services/s3/minio_s3_with_versioning/action.yml index e2db993a86f2..08f06f40fb76 100644 --- a/.github/services/s3/minio_s3_with_versioning/action.yml +++ b/.github/services/s3/minio_s3_with_versioning/action.yml @@ -44,5 +44,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_ENABLE_VERSIONING=true + OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_can_append EOF diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/s3/r2/disabled_action.yml index 8a54f78b5c59..fbad968c9966 100644 --- a/.github/services/s3/r2/disabled_action.yml +++ b/.github/services/s3/r2/disabled_action.yml @@ -40,4 +40,5 @@ runs: OPENDAL_S3_REGION=auto OPENDAL_S3_DELETE_MAX_SIZE=700 OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append EOF diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index 97650ccd02b9..93719fd0f87a 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -448,10 +448,14 @@ impl S3Builder { self } - /// Set bucket versioning status for this backend - pub fn enable_versioning(mut self, enabled: bool) -> Self { - self.config.enable_versioning = enabled; - + /// Deprecated: use [`CapabilityOverrideLayer`][opendal_core::layers::CapabilityOverrideLayer] + /// or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable versioning capability for specific + /// endpoints or test setups. + #[deprecated( + since = "0.57.0", + note = "S3 versioning capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + )] + pub fn enable_versioning(self, _enabled: bool) -> Self { self } @@ -561,9 +565,14 @@ impl S3Builder { self } - /// Enable write with append so that opendal will send write request with append headers. - pub fn enable_write_with_append(mut self) -> Self { - self.config.enable_write_with_append = true; + /// Deprecated: use [`CapabilityOverrideLayer`][opendal_core::layers::CapabilityOverrideLayer] + /// or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable append capability for specific endpoints + /// or test setups. + #[deprecated( + since = "0.57.0", + note = "S3 append capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + )] + pub fn enable_write_with_append(self) -> Self { self } @@ -895,7 +904,7 @@ impl Builder for S3Builder { stat_with_override_content_disposition: !config .disable_stat_with_override, stat_with_override_content_type: !config.disable_stat_with_override, - stat_with_version: config.enable_versioning, + stat_with_version: true, read: true, read_with_if_match: true, @@ -905,12 +914,12 @@ impl Builder for S3Builder { read_with_override_cache_control: true, read_with_override_content_disposition: true, read_with_override_content_type: true, - read_with_version: config.enable_versioning, + read_with_version: true, write: true, write_can_empty: true, write_can_multi: true, - write_can_append: config.enable_write_with_append, + write_can_append: true, write_with_cache_control: true, write_with_content_type: true, @@ -935,7 +944,7 @@ impl Builder for S3Builder { delete: true, delete_max_size: Some(delete_max_size), - delete_with_version: config.enable_versioning, + delete_with_version: true, copy: true, @@ -943,8 +952,8 @@ impl Builder for S3Builder { list_with_limit: true, list_with_start_after: true, list_with_recursive: true, - list_with_versions: config.enable_versioning, - list_with_deleted: config.enable_versioning, + list_with_versions: true, + list_with_deleted: true, presign: true, presign_stat: true, @@ -1282,4 +1291,27 @@ mod tests { "application/json" ); } + + #[allow(deprecated)] + #[test] + fn deprecated_capability_toggles_do_not_change_capability() { + let backend = S3Builder::default() + .bucket("test") + .region("us-east-1") + .allow_anonymous() + .disable_config_load() + .disable_ec2_metadata() + .enable_versioning(false) + .enable_write_with_append() + .build() + .expect("build"); + + let cap = backend.info().full_capability(); + assert!(cap.stat_with_version); + assert!(cap.read_with_version); + assert!(cap.delete_with_version); + assert!(cap.list_with_versions); + assert!(cap.list_with_deleted); + assert!(cap.write_can_append); + } } diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs index a15a2107a421..3db46f2a2c34 100644 --- a/core/services/s3/src/config.rs +++ b/core/services/s3/src/config.rs @@ -42,7 +42,12 @@ pub struct S3Config { /// required. #[serde(alias = "aws_bucket", alias = "aws_bucket_name", alias = "bucket_name")] pub bucket: String, - /// is bucket versioning enabled for this bucket + /// Deprecated: use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to + /// disable versioning capability for specific endpoints or test setups. + #[deprecated( + since = "0.57.0", + note = "S3 versioning capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + )] pub enable_versioning: bool, /// endpoint of this backend. /// @@ -212,7 +217,12 @@ pub struct S3Config { /// For example, Ceph RADOS S3 doesn't support write with if matched. pub disable_write_with_if_match: bool, - /// Enable write with append so that opendal will send write request with append headers. + /// Deprecated: use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to + /// disable append capability for specific endpoints or test setups. + #[deprecated( + since = "0.57.0", + note = "S3 append capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + )] pub enable_write_with_append: bool, /// OpenDAL uses List Objects V2 by default to list objects. diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md index df279f35b7ec..29b8e1ec0444 100644 --- a/core/services/s3/src/docs.md +++ b/core/services/s3/src/docs.md @@ -31,6 +31,8 @@ This service can be used to: - `enable_virtual_host_style`: Enable virtual host style. - `disable_write_with_if_match`: Disable write with if match. - `enable_request_payer`: Enable the request payer for backend. +- `enable_versioning`: Deprecated. S3 versioning capability is enabled by default; use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable it for specific endpoints or test setups. +- `enable_write_with_append`: Deprecated. S3 append capability is enabled by default; use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable it for specific endpoints or test setups. - `default_acl`: Define the default access control list (ACL) when creating a new object. Note that some s3 services like minio do not support this option. Refer to [`S3Builder`]'s public API docs for more information. From 58734dc524f8d3f833bc433c19cdf63753a75a88 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 May 2026 13:37:35 +0800 Subject: [PATCH 3/8] refactor(services/s3): migrate if-match capability toggle --- .../disable_action.yml | 3 +- .../s3/ceph_rados_s3/disable_action.yml | 3 +- core/services/s3/src/backend.rs | 46 +++++-------------- core/services/s3/src/config.rs | 18 ++++---- core/services/s3/src/docs.md | 6 +-- 5 files changed, 25 insertions(+), 51 deletions(-) diff --git a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml index 4aa37ec559bf..3248e61c0c0c 100644 --- a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml +++ b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml @@ -43,6 +43,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on - OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_with_if_match,-write_can_append EOF diff --git a/.github/services/s3/ceph_rados_s3/disable_action.yml b/.github/services/s3/ceph_rados_s3/disable_action.yml index 2d3b51dca6bc..7f703336ba21 100644 --- a/.github/services/s3/ceph_rados_s3/disable_action.yml +++ b/.github/services/s3/ceph_rados_s3/disable_action.yml @@ -41,6 +41,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_with_if_match,-write_can_append EOF diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index 93719fd0f87a..b7be59037eae 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -448,12 +448,10 @@ impl S3Builder { self } - /// Deprecated: use [`CapabilityOverrideLayer`][opendal_core::layers::CapabilityOverrideLayer] - /// or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable versioning capability for specific - /// endpoints or test setups. + /// Deprecated: S3 versioning capability is enabled by default. #[deprecated( since = "0.57.0", - note = "S3 versioning capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + note = "S3 versioning capability is enabled by default and this option is no longer needed." )] pub fn enable_versioning(self, _enabled: bool) -> Self { self @@ -559,18 +557,19 @@ impl S3Builder { self } - /// Disable write with if match so that opendal will not send write request with if match headers. - pub fn disable_write_with_if_match(mut self) -> Self { - self.config.disable_write_with_if_match = true; + /// Deprecated: S3 write with If-Match capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 write with If-Match capability is enabled by default and this option is no longer needed." + )] + pub fn disable_write_with_if_match(self) -> Self { self } - /// Deprecated: use [`CapabilityOverrideLayer`][opendal_core::layers::CapabilityOverrideLayer] - /// or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable append capability for specific endpoints - /// or test setups. + /// Deprecated: S3 append capability is enabled by default. #[deprecated( since = "0.57.0", - note = "S3 append capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + note = "S3 append capability is enabled by default and this option is no longer needed." )] pub fn enable_write_with_append(self) -> Self { self @@ -925,7 +924,7 @@ impl Builder for S3Builder { write_with_content_type: true, write_with_content_disposition: true, write_with_content_encoding: true, - write_with_if_match: !config.disable_write_with_if_match, + write_with_if_match: true, write_with_if_not_exists: true, write_with_user_metadata: true, @@ -1291,27 +1290,4 @@ mod tests { "application/json" ); } - - #[allow(deprecated)] - #[test] - fn deprecated_capability_toggles_do_not_change_capability() { - let backend = S3Builder::default() - .bucket("test") - .region("us-east-1") - .allow_anonymous() - .disable_config_load() - .disable_ec2_metadata() - .enable_versioning(false) - .enable_write_with_append() - .build() - .expect("build"); - - let cap = backend.info().full_capability(); - assert!(cap.stat_with_version); - assert!(cap.read_with_version); - assert!(cap.delete_with_version); - assert!(cap.list_with_versions); - assert!(cap.list_with_deleted); - assert!(cap.write_can_append); - } } diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs index 3db46f2a2c34..54bc9cc1dbdd 100644 --- a/core/services/s3/src/config.rs +++ b/core/services/s3/src/config.rs @@ -42,11 +42,10 @@ pub struct S3Config { /// required. #[serde(alias = "aws_bucket", alias = "aws_bucket_name", alias = "bucket_name")] pub bucket: String, - /// Deprecated: use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to - /// disable versioning capability for specific endpoints or test setups. + /// Deprecated: S3 versioning capability is enabled by default. #[deprecated( since = "0.57.0", - note = "S3 versioning capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + note = "S3 versioning capability is enabled by default and this option is no longer needed." )] pub enable_versioning: bool, /// endpoint of this backend. @@ -212,16 +211,17 @@ pub struct S3Config { /// - "md5" #[serde(alias = "aws_checksum_algorithm")] pub checksum_algorithm: Option, - /// Disable write with if match so that opendal will not send write request with if match headers. - /// - /// For example, Ceph RADOS S3 doesn't support write with if matched. + /// Deprecated: S3 write with If-Match capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 write with If-Match capability is enabled by default and this option is no longer needed." + )] pub disable_write_with_if_match: bool, - /// Deprecated: use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to - /// disable append capability for specific endpoints or test setups. + /// Deprecated: S3 append capability is enabled by default. #[deprecated( since = "0.57.0", - note = "S3 append capability is enabled by default. Use CapabilityOverrideLayer or OPENDAL_TEST_CAPABILITY_OVERRIDES to disable it for specific endpoints or test setups." + note = "S3 append capability is enabled by default and this option is no longer needed." )] pub enable_write_with_append: bool, diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md index 29b8e1ec0444..8b3fc6024c2d 100644 --- a/core/services/s3/src/docs.md +++ b/core/services/s3/src/docs.md @@ -29,10 +29,10 @@ This service can be used to: - `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for backend. - `disable_config_load`: Disable aws config load from env. - `enable_virtual_host_style`: Enable virtual host style. -- `disable_write_with_if_match`: Disable write with if match. - `enable_request_payer`: Enable the request payer for backend. -- `enable_versioning`: Deprecated. S3 versioning capability is enabled by default; use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable it for specific endpoints or test setups. -- `enable_write_with_append`: Deprecated. S3 append capability is enabled by default; use `CapabilityOverrideLayer` or `OPENDAL_TEST_CAPABILITY_OVERRIDES` to disable it for specific endpoints or test setups. +- `disable_write_with_if_match`: Deprecated. S3 write with If-Match capability is enabled by default and this option is no longer needed. +- `enable_versioning`: Deprecated. S3 versioning capability is enabled by default and this option is no longer needed. +- `enable_write_with_append`: Deprecated. S3 append capability is enabled by default and this option is no longer needed. - `default_acl`: Define the default access control list (ACL) when creating a new object. Note that some s3 services like minio do not support this option. Refer to [`S3Builder`]'s public API docs for more information. From 31b2b7db7ab566aea066485886db52bcb3380870 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 May 2026 20:07:15 +0800 Subject: [PATCH 4/8] fix(bindings/dotnet): honor test capability overrides --- .../Behavior/BehaviorOperatorFixture.cs | 111 +++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs index 7401a820a0ca..66b96a220b64 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs @@ -18,13 +18,18 @@ */ using System.Collections; +using System.Globalization; +using System.Reflection; +using System.Text; using OpenDAL.Layer; namespace OpenDAL.Tests; public sealed class BehaviorOperatorFixture : IDisposable { + private const string CapabilityOverridesEnv = "OPENDAL_TEST_CAPABILITY_OVERRIDES"; private readonly Operator? op; + private readonly Capability capability; public string? Scheme { get; } @@ -49,13 +54,14 @@ public BehaviorOperatorFixture() } op = new Operator(scheme, options).WithLayer(new RetryLayer()); + capability = ApplyCapabilityOverrides(op.Info.FullCapability); } public bool IsEnabled => op is not null; public Operator Op => op ?? throw new InvalidOperationException("Behavior operator is not initialized."); - public Capability Capability => Op.Info.FullCapability; + public Capability Capability => capability; public void Dispose() { @@ -112,4 +118,107 @@ private static string BuildRandomRoot(string baseRoot) return $"{trimmed}{Guid.NewGuid():N}/"; } + + private static Capability ApplyCapabilityOverrides(Capability capability) + { + var input = Environment.GetEnvironmentVariable(CapabilityOverridesEnv); + if (string.IsNullOrWhiteSpace(input)) + { + return capability; + } + + foreach (var token in input.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + { + var (name, value) = ParseCapabilityOverride(token); + ApplyCapabilityOverride(ref capability, name, value); + } + + return capability; + } + + private static (string Name, object? Value) ParseCapabilityOverride(string token) + { + if (token.StartsWith('+')) + { + return (token[1..].Trim(), true); + } + + if (token.StartsWith('-')) + { + return (token[1..].Trim(), false); + } + + var parts = token.Split('=', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + throw new InvalidOperationException( + $"invalid {CapabilityOverridesEnv} entry `{token}`: expected `+capability`, `-capability`, or `capability=value`"); + } + + return (parts[0], ParseCapabilityValue(token, parts[1])); + } + + private static object? ParseCapabilityValue(string token, string value) + { + return value.ToLowerInvariant() switch + { + "true" or "on" or "yes" => true, + "false" or "off" or "no" => false, + "none" or "null" or "unset" => null, + _ when ulong.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var result) => result, + _ => throw new InvalidOperationException( + $"invalid {CapabilityOverridesEnv} entry `{token}`: expected a boolean, non-negative integer, or `none`") + }; + } + + private static void ApplyCapabilityOverride(ref Capability capability, string name, object? value) + { + var propertyName = ToCapabilityPropertyName(name); + var property = typeof(Capability).GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public) + ?? throw new InvalidOperationException( + $"invalid {CapabilityOverridesEnv} entry `{name}`: unknown capability"); + + var targetType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; + if (value is null) + { + if (Nullable.GetUnderlyingType(property.PropertyType) is null) + { + throw new InvalidOperationException( + $"invalid {CapabilityOverridesEnv} entry `{name}`: capability does not accept null"); + } + } + else if (value.GetType() != targetType) + { + throw new InvalidOperationException( + $"invalid {CapabilityOverridesEnv} entry `{name}`: capability expects {targetType.Name}"); + } + + var setter = property.GetSetMethod(nonPublic: true) + ?? throw new InvalidOperationException( + $"invalid {CapabilityOverridesEnv} entry `{name}`: capability is not writable"); + + object boxed = capability; + setter.Invoke(boxed, new[] { value }); + capability = (Capability)boxed; + } + + private static string ToCapabilityPropertyName(string name) + { + var builder = new StringBuilder(name.Length); + var upper = true; + + foreach (var c in name.Trim()) + { + if (c == '_') + { + upper = true; + continue; + } + + builder.Append(upper ? char.ToUpperInvariant(c) : c); + upper = false; + } + + return builder.ToString(); + } } From 1ab2437971e1ce748191e4662e98d7131690ac9f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 May 2026 23:19:22 +0800 Subject: [PATCH 5/8] fix(bindings): share capability override layer parsing --- .../Behavior/BehaviorOperatorFixture.cs | 114 +------------ .../OpenDAL/Layer/CapabilityOverrideLayer.cs | 57 +++++++ bindings/dotnet/OpenDAL/NativeMethods.cs | 7 + bindings/dotnet/src/operator.rs | 31 ++++ bindings/java/src/layer.rs | 28 ++++ .../layer/CapabilityOverrideLayer.java | 41 +++++ .../test/behavior/BehaviorExtension.java | 5 + bindings/nodejs/generated.d.ts | 11 ++ bindings/nodejs/index.mjs | 12 +- bindings/nodejs/src/layer.rs | 32 ++++ bindings/nodejs/tests/suites/index.mjs | 5 + bindings/python/python/opendal/layers.pyi | 18 ++ bindings/python/src/layers.rs | 39 +++++ bindings/python/src/lib.rs | 8 +- bindings/python/tests/conftest.py | 7 +- core/Cargo.lock | 1 - core/core/src/layers/capability_override.rs | 155 ++++++++++++++++++ core/testkit/Cargo.toml | 1 - core/testkit/src/utils.rs | 154 +---------------- 19 files changed, 461 insertions(+), 265 deletions(-) create mode 100644 bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs create mode 100644 bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs index 66b96a220b64..d4415d990877 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs @@ -18,9 +18,6 @@ */ using System.Collections; -using System.Globalization; -using System.Reflection; -using System.Text; using OpenDAL.Layer; namespace OpenDAL.Tests; @@ -54,7 +51,13 @@ public BehaviorOperatorFixture() } op = new Operator(scheme, options).WithLayer(new RetryLayer()); - capability = ApplyCapabilityOverrides(op.Info.FullCapability); + var capabilityOverrides = Environment.GetEnvironmentVariable(CapabilityOverridesEnv); + if (!string.IsNullOrWhiteSpace(capabilityOverrides)) + { + op = op.WithLayer(new CapabilityOverrideLayer(capabilityOverrides)); + } + + capability = op.Info.FullCapability; } public bool IsEnabled => op is not null; @@ -118,107 +121,4 @@ private static string BuildRandomRoot(string baseRoot) return $"{trimmed}{Guid.NewGuid():N}/"; } - - private static Capability ApplyCapabilityOverrides(Capability capability) - { - var input = Environment.GetEnvironmentVariable(CapabilityOverridesEnv); - if (string.IsNullOrWhiteSpace(input)) - { - return capability; - } - - foreach (var token in input.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) - { - var (name, value) = ParseCapabilityOverride(token); - ApplyCapabilityOverride(ref capability, name, value); - } - - return capability; - } - - private static (string Name, object? Value) ParseCapabilityOverride(string token) - { - if (token.StartsWith('+')) - { - return (token[1..].Trim(), true); - } - - if (token.StartsWith('-')) - { - return (token[1..].Trim(), false); - } - - var parts = token.Split('=', 2, StringSplitOptions.TrimEntries); - if (parts.Length != 2) - { - throw new InvalidOperationException( - $"invalid {CapabilityOverridesEnv} entry `{token}`: expected `+capability`, `-capability`, or `capability=value`"); - } - - return (parts[0], ParseCapabilityValue(token, parts[1])); - } - - private static object? ParseCapabilityValue(string token, string value) - { - return value.ToLowerInvariant() switch - { - "true" or "on" or "yes" => true, - "false" or "off" or "no" => false, - "none" or "null" or "unset" => null, - _ when ulong.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var result) => result, - _ => throw new InvalidOperationException( - $"invalid {CapabilityOverridesEnv} entry `{token}`: expected a boolean, non-negative integer, or `none`") - }; - } - - private static void ApplyCapabilityOverride(ref Capability capability, string name, object? value) - { - var propertyName = ToCapabilityPropertyName(name); - var property = typeof(Capability).GetProperty(propertyName, BindingFlags.Instance | BindingFlags.Public) - ?? throw new InvalidOperationException( - $"invalid {CapabilityOverridesEnv} entry `{name}`: unknown capability"); - - var targetType = Nullable.GetUnderlyingType(property.PropertyType) ?? property.PropertyType; - if (value is null) - { - if (Nullable.GetUnderlyingType(property.PropertyType) is null) - { - throw new InvalidOperationException( - $"invalid {CapabilityOverridesEnv} entry `{name}`: capability does not accept null"); - } - } - else if (value.GetType() != targetType) - { - throw new InvalidOperationException( - $"invalid {CapabilityOverridesEnv} entry `{name}`: capability expects {targetType.Name}"); - } - - var setter = property.GetSetMethod(nonPublic: true) - ?? throw new InvalidOperationException( - $"invalid {CapabilityOverridesEnv} entry `{name}`: capability is not writable"); - - object boxed = capability; - setter.Invoke(boxed, new[] { value }); - capability = (Capability)boxed; - } - - private static string ToCapabilityPropertyName(string name) - { - var builder = new StringBuilder(name.Length); - var upper = true; - - foreach (var c in name.Trim()) - { - if (c == '_') - { - upper = true; - continue; - } - - builder.Append(upper ? char.ToUpperInvariant(c) : c); - upper = false; - } - - return builder.ToString(); - } } diff --git a/bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs b/bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs new file mode 100644 index 000000000000..04ab049d87aa --- /dev/null +++ b/bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using OpenDAL.Layer.Abstractions; + +namespace OpenDAL.Layer; + +/// +/// Layer that overrides the full capability exposed by an operator. +/// +public sealed class CapabilityOverrideLayer : ILayer +{ + /// + /// Creates a capability override layer. + /// + /// Comma-separated capability override entries. + public CapabilityOverrideLayer(string overrides) + { + ArgumentException.ThrowIfNullOrWhiteSpace(overrides); + Overrides = overrides; + } + + /// + /// Gets capability override entries. + /// + public string Overrides { get; } + + /// + /// Applies capability overrides to the specified operator. + /// + /// Operator to layer. + /// The layered operator instance. + public Operator Apply(Operator op) + { + ArgumentNullException.ThrowIfNull(op); + ObjectDisposedException.ThrowIf(op.IsInvalid, op); + + var result = NativeMethods.operator_layer_capability_override(op, Overrides); + return op.ApplyLayerResult(result); + } +} diff --git a/bindings/dotnet/OpenDAL/NativeMethods.cs b/bindings/dotnet/OpenDAL/NativeMethods.cs index 609408fedc26..fd8ecb3ad41a 100644 --- a/bindings/dotnet/OpenDAL/NativeMethods.cs +++ b/bindings/dotnet/OpenDAL/NativeMethods.cs @@ -154,6 +154,13 @@ internal static partial OpenDALOperatorResult operator_layer_concurrent_limit( nuint permits ); + [LibraryImport(__DllName, EntryPoint = "operator_layer_capability_override", StringMarshalling = StringMarshalling.Utf8)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial OpenDALOperatorResult operator_layer_capability_override( + Operator op, + string overrides + ); + [LibraryImport(__DllName, EntryPoint = "operator_layer_timeout")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial OpenDALOperatorResult operator_layer_timeout( diff --git a/bindings/dotnet/src/operator.rs b/bindings/dotnet/src/operator.rs index af725a3a23cc..a66f6b9f9581 100644 --- a/bindings/dotnet/src/operator.rs +++ b/bindings/dotnet/src/operator.rs @@ -410,6 +410,37 @@ fn operator_layer_concurrent_limit_inner( Ok(Box::into_raw(Box::new(op.clone().layer(concurrent_limit))) as *mut c_void) } +/// Create a new operator layered with capability override behavior. +/// +/// The current operator is not modified. Returned pointer must be released with +/// `operator_free`. +/// # Safety +/// +/// - `op` must be a valid operator pointer from `operator_construct`. +/// - `overrides` must be a valid null-terminated UTF-8 string. +#[unsafe(no_mangle)] +pub extern "C" fn operator_layer_capability_override( + op: *const opendal::Operator, + overrides: *const c_char, +) -> OpendalOperatorResult { + match operator_layer_capability_override_inner(op, overrides) { + Ok(value) => OpendalOperatorResult::ok(value), + Err(error) => OpendalOperatorResult::from_error(error), + } +} + +fn operator_layer_capability_override_inner( + op: *const opendal::Operator, + overrides: *const c_char, +) -> Result<*mut c_void, OpenDALError> { + let op = require_operator(op)?; + let overrides = require_cstr(overrides, "capability overrides")?; + let layer = opendal::layers::CapabilityOverrideLayer::from_overrides(overrides) + .map_err(OpenDALError::from_opendal_error)?; + + Ok(Box::into_raw(Box::new(op.clone().layer(layer))) as *mut c_void) +} + /// Create a new operator layered with timeout behavior. /// /// The current operator is not modified. Returned pointer must be released with diff --git a/bindings/java/src/layer.rs b/bindings/java/src/layer.rs index c649bfa0286b..5f41567c8c9a 100644 --- a/bindings/java/src/layer.rs +++ b/bindings/java/src/layer.rs @@ -17,12 +17,16 @@ use std::time::Duration; +use crate::Result; +use crate::convert::jstring_to_string; use jni::JNIEnv; use jni::objects::JClass; +use jni::objects::JString; use jni::sys::jboolean; use jni::sys::jfloat; use jni::sys::jlong; use opendal::Operator; +use opendal::layers::CapabilityOverrideLayer; use opendal::layers::ConcurrentLimitLayer; use opendal::layers::RetryLayer; @@ -51,6 +55,30 @@ pub extern "system" fn Java_org_apache_opendal_layer_RetryLayer_doLayer( Box::into_raw(Box::new(op.clone().layer(retry))) as jlong } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_apache_opendal_layer_CapabilityOverrideLayer_doLayer( + mut env: JNIEnv, + _: JClass, + op: *mut Operator, + overrides: JString, +) -> jlong { + intern_capability_override_layer(&mut env, op, overrides).unwrap_or_else(|e| { + e.throw(&mut env); + 0 + }) +} + +fn intern_capability_override_layer( + env: &mut JNIEnv, + op: *mut Operator, + overrides: JString, +) -> Result { + let op = unsafe { &*op }; + let overrides = jstring_to_string(env, &overrides)?; + let layer = CapabilityOverrideLayer::from_overrides(&overrides)?; + Ok(Box::into_raw(Box::new(op.clone().layer(layer))) as jlong) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_org_apache_opendal_layer_ConcurrentLimitLayer_doLayer( _: JNIEnv, diff --git a/bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java b/bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java new file mode 100644 index 000000000000..ace54af04028 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.layer; + +import org.apache.opendal.Layer; + +/** + * Layer that overrides the full capability exposed by an operator. + */ +public class CapabilityOverrideLayer extends Layer { + + private final String overrides; + + public CapabilityOverrideLayer(String overrides) { + this.overrides = overrides; + } + + @Override + protected long layer(long nativeOp) { + return doLayer(nativeOp, overrides); + } + + private static native long doLayer(long nativeHandle, String overrides); +} diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java index 367e79c66715..3ad216a6acf9 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java @@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.opendal.AsyncOperator; import org.apache.opendal.Operator; +import org.apache.opendal.layer.CapabilityOverrideLayer; import org.apache.opendal.layer.RetryLayer; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; @@ -67,6 +68,10 @@ public void beforeAll(ExtensionContext context) { @Cleanup final AsyncOperator op = AsyncOperator.of(scheme.toLowerCase().replace('_', '-'), config); this.asyncOperator = op.layer(RetryLayer.builder().build()); + final String capabilityOverrides = dotenv.get("OPENDAL_TEST_CAPABILITY_OVERRIDES"); + if (capabilityOverrides != null && !capabilityOverrides.trim().isEmpty()) { + this.asyncOperator = this.asyncOperator.layer(new CapabilityOverrideLayer(capabilityOverrides)); + } this.operator = this.asyncOperator.blocking(); this.scheme = scheme; diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts index 999bb81471b7..8ac5619d1bac 100644 --- a/bindings/nodejs/generated.d.ts +++ b/bindings/nodejs/generated.d.ts @@ -207,6 +207,17 @@ export declare class Capability { get shared(): boolean } +/** + * Capability override layer + * + * Override the full capability exposed by an operator. + */ +export declare class CapabilityOverrideLayer { + /** Create a new CapabilityOverrideLayer from capability override entries. */ + constructor(overrides: string) + build(): ExternalObject +} + /** * Concurrent limit layer * diff --git a/bindings/nodejs/index.mjs b/bindings/nodejs/index.mjs index 1eac4ec77dca..59b0ee47eede 100644 --- a/bindings/nodejs/index.mjs +++ b/bindings/nodejs/index.mjs @@ -120,7 +120,16 @@ class BlockingWriteStream extends Writable { } import * as generated from './generated.js' -const { Operator, RetryLayer, ConcurrentLimitLayer, BlockingReader, Reader, BlockingWriter, Writer } = generated +const { + Operator, + CapabilityOverrideLayer, + RetryLayer, + ConcurrentLimitLayer, + BlockingReader, + Reader, + BlockingWriter, + Writer, +} = generated BlockingReader.prototype.createReadStream = function (options) { return new BlockingReadStream(this, options) @@ -139,6 +148,7 @@ Writer.prototype.createWriteStream = function (options) { } export const layers = { + CapabilityOverrideLayer, RetryLayer, ConcurrentLimitLayer, } diff --git a/bindings/nodejs/src/layer.rs b/bindings/nodejs/src/layer.rs index ff6e9dd2bb81..b78e458b7193 100644 --- a/bindings/nodejs/src/layer.rs +++ b/bindings/nodejs/src/layer.rs @@ -18,6 +18,7 @@ use std::time::Duration; use napi::bindgen_prelude::External; +use napi::bindgen_prelude::Result; pub trait NodeLayer: Send + Sync { fn layer(&self, op: opendal::Operator) -> opendal::Operator; @@ -35,6 +36,37 @@ impl NodeLayer for opendal::layers::RetryLayer { } } +impl NodeLayer for opendal::layers::CapabilityOverrideLayer { + fn layer(&self, op: opendal::Operator) -> opendal::Operator { + op.layer(self.clone()) + } +} + +/// Capability override layer +/// +/// Override the full capability exposed by an operator. +#[napi] +pub struct CapabilityOverrideLayer { + overrides: String, +} + +#[napi] +impl CapabilityOverrideLayer { + /// Create a new CapabilityOverrideLayer from capability override entries. + #[napi(constructor)] + pub fn new(overrides: String) -> Self { + Self { overrides } + } + + #[napi] + pub fn build(&self) -> Result> { + let l = opendal::layers::CapabilityOverrideLayer::from_overrides(&self.overrides) + .map_err(|err| napi::Error::from_reason(format!("{err}")))?; + + Ok(External::new(Layer { inner: Box::new(l) })) + } +} + /// Retry layer /// /// Add retry for temporary failed operations. diff --git a/bindings/nodejs/tests/suites/index.mjs b/bindings/nodejs/tests/suites/index.mjs index 36d7de093698..152ebcf7d5bb 100644 --- a/bindings/nodejs/tests/suites/index.mjs +++ b/bindings/nodejs/tests/suites/index.mjs @@ -61,6 +61,11 @@ export function runner(testName, scheme) { retryLayer.maxTimes = 4 operator = operator.layer(retryLayer.build()) + if (process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES) { + operator = operator.layer( + new layers.CapabilityOverrideLayer(process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES).build() + ) + } describe.skipIf(!operator)(testName, () => { AsyncIOTestRun(operator) diff --git a/bindings/python/python/opendal/layers.pyi b/bindings/python/python/opendal/layers.pyi index d46e6f5e18eb..48862d1f513f 100644 --- a/bindings/python/python/opendal/layers.pyi +++ b/bindings/python/python/opendal/layers.pyi @@ -21,6 +21,24 @@ import builtins import typing +@typing.final +class CapabilityOverrideLayer(Layer): + r"""A layer that overrides the full capability exposed by an operator.""" + + def __new__(cls, overrides: builtins.str) -> CapabilityOverrideLayer: + r""" + Create a new CapabilityOverrideLayer from capability override entries. + + Parameters + ---------- + overrides : str + Comma-separated capability override entries. + + Returns + ------- + CapabilityOverrideLayer + """ + @typing.final class ConcurrentLimitLayer(Layer): r""" diff --git a/bindings/python/src/layers.rs b/bindings/python/src/layers.rs index 84365f88e530..92957e7ec2f8 100644 --- a/bindings/python/src/layers.rs +++ b/bindings/python/src/layers.rs @@ -29,6 +29,45 @@ pub trait PythonLayer: Send + Sync { #[pyclass(module = "opendal.layers", subclass)] pub struct Layer(pub Box); +/// A layer that overrides the full capability exposed by an operator. +#[gen_stub_pyclass] +#[pyclass(module = "opendal.layers", extends=Layer)] +#[derive(Clone)] +pub struct CapabilityOverrideLayer(ocore::layers::CapabilityOverrideLayer); + +impl PythonLayer for CapabilityOverrideLayer { + fn layer(&self, op: Operator) -> Operator { + op.layer(self.0.clone()) + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl CapabilityOverrideLayer { + /// Create a new CapabilityOverrideLayer from capability override entries. + /// + /// Parameters + /// ---------- + /// overrides : str + /// Comma-separated capability override entries. + /// + /// Returns + /// ------- + /// CapabilityOverrideLayer + #[gen_stub(override_return_type(type_repr = "CapabilityOverrideLayer"))] + #[new] + #[pyo3(signature = (overrides))] + fn new(overrides: &str) -> PyResult> { + let layer = Self( + ocore::layers::CapabilityOverrideLayer::from_overrides(overrides) + .map_err(format_pyerr)?, + ); + let class = PyClassInitializer::from(Layer(Box::new(layer.clone()))).add_subclass(layer); + + Ok(class) + } +} + /// A layer that retries operations that fail with temporary errors. /// /// Operations are retried if they fail with an error for which diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index bf413eb7501f..7f9a22148693 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -64,7 +64,13 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { py, m, "layers", - [Layer, RetryLayer, ConcurrentLimitLayer, MimeGuessLayer] + [ + Layer, + CapabilityOverrideLayer, + RetryLayer, + ConcurrentLimitLayer, + MimeGuessLayer + ] )?; // Types module diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index 5bf32d54588c..3c89efcc2d9f 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -25,6 +25,7 @@ load_dotenv() pytest_plugins = ("pytest_asyncio",) +CAPABILITY_OVERRIDES_ENV = "OPENDAL_TEST_CAPABILITY_OVERRIDES" def pytest_configure(config): @@ -59,12 +60,16 @@ def setup_config(service_name): @pytest.fixture(scope="session") def async_operator(service_name, setup_config): - return ( + operator = ( opendal.AsyncOperator(service_name, **setup_config) .layer(opendal.layers.RetryLayer()) .layer(opendal.layers.ConcurrentLimitLayer(1024)) .layer(opendal.layers.MimeGuessLayer()) ) + overrides = os.environ.get(CAPABILITY_OVERRIDES_ENV) + if overrides: + operator = operator.layer(opendal.layers.CapabilityOverrideLayer(overrides)) + return operator @pytest.fixture(scope="session") diff --git a/core/Cargo.lock b/core/Cargo.lock index fd9e54078a90..9c6aac285c01 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -7560,7 +7560,6 @@ dependencies = [ "opendal-layer-retry", "opendal-layer-timeout", "rand 0.10.1", - "serde_json", "sha2 0.11.0", "tokio", "uuid", diff --git a/core/core/src/layers/capability_override.rs b/core/core/src/layers/capability_override.rs index 65cb2e0a1a37..c44c2cef234c 100644 --- a/core/core/src/layers/capability_override.rs +++ b/core/core/src/layers/capability_override.rs @@ -18,6 +18,10 @@ use std::fmt; use std::sync::Arc; +use serde_json::Map; +use serde_json::Number; +use serde_json::Value; + use crate::raw::*; use crate::*; @@ -67,6 +71,19 @@ impl CapabilityOverrideLayer { apply: Arc::new(apply), } } + + /// Create a new [`CapabilityOverrideLayer`] from capability override entries. + /// + /// The input is a comma-separated list of capability assignments: + /// + /// - `+read` sets a boolean capability to `true` + /// - `-read` sets a boolean capability to `false` + /// - `delete_max_size=1000` sets a numeric capability + /// - `delete_max_size=none` unsets an optional capability + pub fn from_overrides(input: &str) -> Result { + let overrides = CapabilityOverrides::parse(input)?; + Ok(Self::new(move |cap| overrides.apply(cap))) + } } impl Layer for CapabilityOverrideLayer { @@ -80,6 +97,102 @@ impl Layer for CapabilityOverrideLayer { } } +#[derive(Clone, Debug, Default)] +struct CapabilityOverrides { + values: Map, +} + +impl CapabilityOverrides { + fn parse(input: &str) -> Result { + let mut overrides = Self::default(); + + for token in input.split(',').map(str::trim).filter(|v| !v.is_empty()) { + let (name, value) = parse_capability_override(token)?; + overrides.values.insert(name.to_string(), value); + overrides.try_apply(Capability::default()).map_err(|err| { + invalid_capability_override(token, &format!("failed to apply override: {err}")) + })?; + } + + Ok(overrides) + } + + fn apply(&self, cap: Capability) -> Capability { + self.try_apply(cap) + .expect("capability overrides must be validated before applying") + } + + fn try_apply(&self, cap: Capability) -> Result { + let mut value = serde_json::to_value(cap).map_err(|err| { + Error::new( + ErrorKind::Unexpected, + format!("failed to serialize capability: {err}"), + ) + })?; + let object = value.as_object_mut().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "serialized capability must be a JSON object", + ) + })?; + object.extend(self.values.clone()); + + serde_json::from_value(value).map_err(|err| { + Error::new( + ErrorKind::ConfigInvalid, + format!("failed to deserialize capability overrides: {err}"), + ) + }) + } +} + +fn parse_capability_override(token: &str) -> Result<(&str, Value)> { + if let Some(name) = token.strip_prefix('+') { + return Ok((name.trim(), Value::Bool(true))); + } + + if let Some(name) = token.strip_prefix('-') { + return Ok((name.trim(), Value::Bool(false))); + } + + let Some((name, value)) = token.split_once('=') else { + return Err(invalid_capability_override( + token, + "expected `+capability`, `-capability`, or `capability=value`", + )); + }; + + Ok(( + name.trim(), + parse_capability_value(value.trim()) + .map_err(|err| invalid_capability_override(token, &err.to_string()))?, + )) +} + +fn invalid_capability_override(token: &str, reason: &str) -> Error { + Error::new( + ErrorKind::ConfigInvalid, + format!("invalid capability override entry `{token}`: {reason}"), + ) +} + +fn parse_capability_value(value: &str) -> Result { + match value { + "true" | "on" | "yes" => Ok(Value::Bool(true)), + "false" | "off" | "no" => Ok(Value::Bool(false)), + "none" | "null" | "unset" => Ok(Value::Null), + _ => value + .parse::() + .map(|v| Value::Number(Number::from(v))) + .map_err(|_| { + Error::new( + ErrorKind::ConfigInvalid, + "expected a boolean, non-negative integer, or `none`", + ) + }), + } +} + #[cfg(test)] mod tests { use super::*; @@ -107,4 +220,46 @@ mod tests { Ok(()) } + + #[test] + fn parse_capability_overrides() -> Result<()> { + let layer = + CapabilityOverrideLayer::from_overrides("-read,+write_can_append,delete_max_size=7")?; + let op = Operator::new(services::Memory::default())? + .layer(layer) + .finish(); + + assert!(!op.info().full_capability().read); + assert!(op.info().full_capability().write_can_append); + assert_eq!(op.info().full_capability().delete_max_size, Some(7)); + + Ok(()) + } + + #[test] + fn parse_bool_assignments_and_unset_sizes() -> Result<()> { + let layer = + CapabilityOverrideLayer::from_overrides("read=false,write=true,delete_max_size=none")?; + let op = Operator::new(services::Memory::default())? + .layer(layer) + .finish(); + + assert!(!op.info().full_capability().read); + assert!(op.info().full_capability().write); + assert_eq!(op.info().full_capability().delete_max_size, None); + + Ok(()) + } + + #[test] + fn reject_unknown_capability() { + let err = CapabilityOverrideLayer::from_overrides("-not_a_capability").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reject_invalid_capability_type() { + let err = CapabilityOverrideLayer::from_overrides("read=1").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } } diff --git a/core/testkit/Cargo.toml b/core/testkit/Cargo.toml index a6c2396d70c4..537964a84633 100644 --- a/core/testkit/Cargo.toml +++ b/core/testkit/Cargo.toml @@ -38,7 +38,6 @@ opendal-layer-logging = { path = "../layers/logging", version = "0.56.0", defaul opendal-layer-retry = { path = "../layers/retry", version = "0.56.0", default-features = false } opendal-layer-timeout = { path = "../layers/timeout", version = "0.56.0", default-features = false } rand = { workspace = true } -serde_json = { workspace = true } sha2 = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"] } uuid = { workspace = true } diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs index 01c219ebed6a..3d53ee4cbeff 100644 --- a/core/testkit/src/utils.rs +++ b/core/testkit/src/utils.rs @@ -19,18 +19,12 @@ use std::collections::HashMap; use std::env; use std::sync::LazyLock; -use opendal_core::Capability; -use opendal_core::Error; -use opendal_core::ErrorKind; use opendal_core::Operator; use opendal_core::Result; use opendal_core::layers::CapabilityOverrideLayer; use opendal_layer_logging::LoggingLayer; use opendal_layer_retry::RetryLayer; use opendal_layer_timeout::TimeoutLayer; -use serde_json::Map; -use serde_json::Number; -use serde_json::Value; use sha2::Digest; use sha2::Sha256; @@ -98,10 +92,7 @@ pub fn init_test_service() -> Result> { let mut op = Operator::via_iter(scheme, cfg).expect("must succeed"); if let Ok(overrides) = env::var(OPENDAL_TEST_CAPABILITY_OVERRIDES) { - let overrides = CapabilityOverrides::parse(&overrides)?; - op = op.layer(CapabilityOverrideLayer::new(move |cap| { - overrides.apply(cap) - })); + op = op.layer(CapabilityOverrideLayer::from_overrides(&overrides)?); } let op = op @@ -111,146 +102,3 @@ pub fn init_test_service() -> Result> { Ok(Some(op)) } - -#[derive(Clone, Debug, Default)] -struct CapabilityOverrides { - values: Map, -} - -impl CapabilityOverrides { - fn parse(input: &str) -> Result { - let mut overrides = Self::default(); - - for token in input.split(',').map(str::trim).filter(|v| !v.is_empty()) { - let (name, value) = parse_capability_override(token)?; - overrides.values.insert(name.to_string(), value); - overrides.try_apply(Capability::default()).map_err(|err| { - invalid_capability_override(token, &format!("failed to apply override: {err}")) - })?; - } - - Ok(overrides) - } - - fn apply(&self, cap: Capability) -> Capability { - self.try_apply(cap) - .expect("capability overrides must be validated before applying") - } - - fn try_apply(&self, cap: Capability) -> Result { - let mut value = serde_json::to_value(cap).map_err(|err| { - Error::new( - ErrorKind::Unexpected, - format!("failed to serialize capability: {err}"), - ) - })?; - let object = value.as_object_mut().ok_or_else(|| { - Error::new( - ErrorKind::Unexpected, - "serialized capability must be a JSON object", - ) - })?; - object.extend(self.values.clone()); - - serde_json::from_value(value).map_err(|err| { - Error::new( - ErrorKind::ConfigInvalid, - format!("failed to deserialize capability overrides: {err}"), - ) - }) - } -} - -fn parse_capability_override(token: &str) -> Result<(&str, Value)> { - if let Some(name) = token.strip_prefix('+') { - return Ok((name.trim(), Value::Bool(true))); - } - - if let Some(name) = token.strip_prefix('-') { - return Ok((name.trim(), Value::Bool(false))); - } - - let Some((name, value)) = token.split_once('=') else { - return Err(invalid_capability_override( - token, - "expected `+capability`, `-capability`, or `capability=value`", - )); - }; - - Ok(( - name.trim(), - parse_capability_value(value.trim()) - .map_err(|err| invalid_capability_override(token, &err.to_string()))?, - )) -} - -fn invalid_capability_override(token: &str, reason: &str) -> Error { - Error::new( - ErrorKind::ConfigInvalid, - format!("invalid {OPENDAL_TEST_CAPABILITY_OVERRIDES} entry `{token}`: {reason}"), - ) -} - -fn parse_capability_value(value: &str) -> Result { - match value { - "true" | "on" | "yes" => Ok(Value::Bool(true)), - "false" | "off" | "no" => Ok(Value::Bool(false)), - "none" | "null" | "unset" => Ok(Value::Null), - _ => value - .parse::() - .map(|v| Value::Number(Number::from(v))) - .map_err(|_| { - Error::new( - ErrorKind::ConfigInvalid, - "expected a boolean, non-negative integer, or `none`", - ) - }), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_capability_overrides() { - let overrides = CapabilityOverrides::parse("-read,+write_can_append,delete_max_size=7") - .expect("override must parse"); - let cap = overrides.apply(Capability { - read: true, - delete_max_size: Some(1000), - ..Default::default() - }); - - assert!(!cap.read); - assert!(cap.write_can_append); - assert_eq!(cap.delete_max_size, Some(7)); - } - - #[test] - fn parse_bool_assignments_and_unset_sizes() { - let overrides = CapabilityOverrides::parse("read=false,write=true,delete_max_size=none") - .expect("override must parse"); - let cap = overrides.apply(Capability { - read: true, - delete_max_size: Some(1000), - ..Default::default() - }); - - assert!(!cap.read); - assert!(cap.write); - assert_eq!(cap.delete_max_size, None); - } - - #[test] - fn reject_unknown_capability() { - let err = CapabilityOverrides::parse("-not_a_capability").unwrap_err(); - assert_eq!(err.kind(), ErrorKind::ConfigInvalid); - } - - #[test] - fn reject_invalid_capability_type() { - let err = CapabilityOverrides::parse("read=1").unwrap_err(); - assert_eq!(err.kind(), ErrorKind::ConfigInvalid); - } -} From 82436567af91fce29beee3314b82466509e8aeeb Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 12 May 2026 23:29:09 +0800 Subject: [PATCH 6/8] refactor(core): require assignment capability overrides --- .github/services/s3/0_minio_s3/action.yml | 2 +- .github/services/s3/aws_s3/action.yml | 2 +- .../s3/aws_s3_with_list_objects_v1/action.yml | 2 +- .../services/s3/aws_s3_with_sse_c/action.yml | 2 +- .../s3/aws_s3_with_versioning/action.yml | 2 +- .../s3/aws_s3_with_virtual_host/action.yml | 2 +- .../disable_action.yml | 2 +- .../s3/ceph_rados_s3/disable_action.yml | 2 +- .../s3/minio_s3_with_anonymous/action.yml | 2 +- .../minio_s3_with_list_objects_v1/action.yml | 2 +- .../s3/minio_s3_with_versioning/action.yml | 2 +- .github/services/s3/r2/disabled_action.yml | 2 +- core/core/src/layers/capability_override.rs | 27 +++++++++---------- core/tests/behavior/README.md | 4 +-- 14 files changed, 27 insertions(+), 28 deletions(-) diff --git a/.github/services/s3/0_minio_s3/action.yml b/.github/services/s3/0_minio_s3/action.yml index 3ae887e301cf..c2c10eaea174 100644 --- a/.github/services/s3/0_minio_s3/action.yml +++ b/.github/services/s3/0_minio_s3/action.yml @@ -43,5 +43,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/aws_s3/action.yml b/.github/services/s3/aws_s3/action.yml index 974a4b57d90a..a45126876021 100644 --- a/.github/services/s3/aws_s3/action.yml +++ b/.github/services/s3/aws_s3/action.yml @@ -36,4 +36,4 @@ runs: - name: Add capability overrides shell: bash run: | - echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_list_objects_v1/action.yml b/.github/services/s3/aws_s3_with_list_objects_v1/action.yml index feb14ab57ad8..5008dee6f042 100644 --- a/.github/services/s3/aws_s3_with_list_objects_v1/action.yml +++ b/.github/services/s3/aws_s3_with_list_objects_v1/action.yml @@ -37,4 +37,4 @@ runs: shell: bash run: | echo "OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true" >> $GITHUB_ENV - echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml index bdb11c817ca0..2dde7218433a 100644 --- a/.github/services/s3/aws_s3_with_sse_c/action.yml +++ b/.github/services/s3/aws_s3_with_sse_c/action.yml @@ -40,5 +40,5 @@ runs: OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256 OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA= OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw== - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/aws_s3_with_versioning/action.yml b/.github/services/s3/aws_s3_with_versioning/action.yml index 8a36e05cdd0c..c1f85c397f21 100644 --- a/.github/services/s3/aws_s3_with_versioning/action.yml +++ b/.github/services/s3/aws_s3_with_versioning/action.yml @@ -36,4 +36,4 @@ runs: - name: Add capability overrides shell: bash run: | - echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_can_append" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_virtual_host/action.yml b/.github/services/s3/aws_s3_with_virtual_host/action.yml index 9ee8ec3eba44..025eff07141b 100644 --- a/.github/services/s3/aws_s3_with_virtual_host/action.yml +++ b/.github/services/s3/aws_s3_with_virtual_host/action.yml @@ -37,4 +37,4 @@ runs: shell: bash run: | echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV - echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml index 3248e61c0c0c..e0169e408bab 100644 --- a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml +++ b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml @@ -43,5 +43,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_with_if_match,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=write_with_if_match=false,write_can_append=false EOF diff --git a/.github/services/s3/ceph_rados_s3/disable_action.yml b/.github/services/s3/ceph_rados_s3/disable_action.yml index 7f703336ba21..ea7d7b45d7cd 100644 --- a/.github/services/s3/ceph_rados_s3/disable_action.yml +++ b/.github/services/s3/ceph_rados_s3/disable_action.yml @@ -41,5 +41,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_with_if_match,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_with_if_match=false,write_can_append=false EOF diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml index 70aeea083f6f..0807d2db42fe 100644 --- a/.github/services/s3/minio_s3_with_anonymous/action.yml +++ b/.github/services/s3/minio_s3_with_anonymous/action.yml @@ -49,5 +49,5 @@ runs: OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_ALLOW_ANONYMOUS=on OPENDAL_S3_DISABLE_EC2_METADATA=on - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/minio_s3_with_list_objects_v1/action.yml b/.github/services/s3/minio_s3_with_list_objects_v1/action.yml index 98dc9ae406f8..2c8bb3c491a4 100644 --- a/.github/services/s3/minio_s3_with_list_objects_v1/action.yml +++ b/.github/services/s3/minio_s3_with_list_objects_v1/action.yml @@ -44,5 +44,5 @@ runs: OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/minio_s3_with_versioning/action.yml b/.github/services/s3/minio_s3_with_versioning/action.yml index 08f06f40fb76..1f838a50017f 100644 --- a/.github/services/s3/minio_s3_with_versioning/action.yml +++ b/.github/services/s3/minio_s3_with_versioning/action.yml @@ -44,5 +44,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 - OPENDAL_TEST_CAPABILITY_OVERRIDES=-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false EOF diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/s3/r2/disabled_action.yml index fbad968c9966..fee1b3988f2b 100644 --- a/.github/services/s3/r2/disabled_action.yml +++ b/.github/services/s3/r2/disabled_action.yml @@ -40,5 +40,5 @@ runs: OPENDAL_S3_REGION=auto OPENDAL_S3_DELETE_MAX_SIZE=700 OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true - OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,-delete_with_version,-list_with_versions,-list_with_deleted,-write_can_append + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/core/core/src/layers/capability_override.rs b/core/core/src/layers/capability_override.rs index c44c2cef234c..5f3dd97cb1d3 100644 --- a/core/core/src/layers/capability_override.rs +++ b/core/core/src/layers/capability_override.rs @@ -76,8 +76,8 @@ impl CapabilityOverrideLayer { /// /// The input is a comma-separated list of capability assignments: /// - /// - `+read` sets a boolean capability to `true` - /// - `-read` sets a boolean capability to `false` + /// - `read=true` sets a boolean capability to `true` + /// - `read=false` sets a boolean capability to `false` /// - `delete_max_size=1000` sets a numeric capability /// - `delete_max_size=none` unsets an optional capability pub fn from_overrides(input: &str) -> Result { @@ -147,18 +147,10 @@ impl CapabilityOverrides { } fn parse_capability_override(token: &str) -> Result<(&str, Value)> { - if let Some(name) = token.strip_prefix('+') { - return Ok((name.trim(), Value::Bool(true))); - } - - if let Some(name) = token.strip_prefix('-') { - return Ok((name.trim(), Value::Bool(false))); - } - let Some((name, value)) = token.split_once('=') else { return Err(invalid_capability_override( token, - "expected `+capability`, `-capability`, or `capability=value`", + "expected `capability=value`", )); }; @@ -223,8 +215,9 @@ mod tests { #[test] fn parse_capability_overrides() -> Result<()> { - let layer = - CapabilityOverrideLayer::from_overrides("-read,+write_can_append,delete_max_size=7")?; + let layer = CapabilityOverrideLayer::from_overrides( + "read=false,write_can_append=true,delete_max_size=7", + )?; let op = Operator::new(services::Memory::default())? .layer(layer) .finish(); @@ -253,7 +246,13 @@ mod tests { #[test] fn reject_unknown_capability() { - let err = CapabilityOverrideLayer::from_overrides("-not_a_capability").unwrap_err(); + let err = CapabilityOverrideLayer::from_overrides("not_a_capability=false").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reject_capability_shorthand() { + let err = CapabilityOverrideLayer::from_overrides("-read").unwrap_err(); assert_eq!(err.kind(), ErrorKind::ConfigInvalid); } diff --git a/core/tests/behavior/README.md b/core/tests/behavior/README.md index 84bf714545f1..5eb36d56a748 100644 --- a/core/tests/behavior/README.md +++ b/core/tests/behavior/README.md @@ -37,10 +37,10 @@ Notice: If the env variables are not set, all behavior tests will be skipped by Behavior tests are selected by the operator's full capability. Test setups can override capability with `OPENDAL_TEST_CAPABILITY_OVERRIDES`: ```shell -OPENDAL_TEST_CAPABILITY_OVERRIDES=-stat_with_version,-read_with_version,delete_max_size=700 +OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_max_size=700 ``` -Use `-capability` to disable a boolean capability, `+capability` to enable one, and `capability=value` to set boolean or numeric capability values. +Use `capability=false` or `capability=true` to set boolean capabilities, and `capability=value` to set numeric capability values. ## Run From 646e25921448fd2bfce84394430e351048c0913f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 18 May 2026 00:26:07 +0800 Subject: [PATCH 7/8] fix(bindings/nodejs): update capability override layer exports --- bindings/nodejs/generated.js | 1 + bindings/nodejs/tests/suites/index.mjs | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bindings/nodejs/generated.js b/bindings/nodejs/generated.js index dc74d0e045b6..612c1dfa0193 100644 --- a/bindings/nodejs/generated.js +++ b/bindings/nodejs/generated.js @@ -531,6 +531,7 @@ module.exports.BlockingLister = nativeBinding.BlockingLister module.exports.BlockingReader = nativeBinding.BlockingReader module.exports.BlockingWriter = nativeBinding.BlockingWriter module.exports.Capability = nativeBinding.Capability +module.exports.CapabilityOverrideLayer = nativeBinding.CapabilityOverrideLayer module.exports.ConcurrentLimitLayer = nativeBinding.ConcurrentLimitLayer module.exports.Entry = nativeBinding.Entry module.exports.Layer = nativeBinding.Layer diff --git a/bindings/nodejs/tests/suites/index.mjs b/bindings/nodejs/tests/suites/index.mjs index 152ebcf7d5bb..36794e62c683 100644 --- a/bindings/nodejs/tests/suites/index.mjs +++ b/bindings/nodejs/tests/suites/index.mjs @@ -62,9 +62,7 @@ export function runner(testName, scheme) { operator = operator.layer(retryLayer.build()) if (process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES) { - operator = operator.layer( - new layers.CapabilityOverrideLayer(process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES).build() - ) + operator = operator.layer(new layers.CapabilityOverrideLayer(process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES).build()) } describe.skipIf(!operator)(testName, () => { From 44d1ee2d276cdc3cca493b1ff231aa4c72b9b6de Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 18 May 2026 00:33:22 +0800 Subject: [PATCH 8/8] chore(bindings): refresh generated service config docs --- .../org/apache/opendal/ServiceConfig.java | 13 +++++++--- bindings/python/src/services.rs | 26 +++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java index 52175686d3da..7d337195527b 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java +++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java @@ -2910,8 +2910,9 @@ class S3 implements ServiceConfig { */ public final Boolean disableStatWithOverride; /** - *

Disable write with if match so that opendal will not send write request with if match headers.

- *

For example, Ceph RADOS S3 doesn't support write with if matched.

+ *

Deprecated: S3 write with If-Match capability is enabled by default.

+ * + * @deprecated S3 write with If-Match capability is enabled by default and this option is no longer needed. */ public final Boolean disableWriteWithIfMatch; /** @@ -2919,7 +2920,9 @@ class S3 implements ServiceConfig { */ public final Boolean enableRequestPayer; /** - *

is bucket versioning enabled for this bucket

+ *

Deprecated: S3 versioning capability is enabled by default.

+ * + * @deprecated S3 versioning capability is enabled by default and this option is no longer needed. */ public final Boolean enableVersioning; /** @@ -2932,7 +2935,9 @@ class S3 implements ServiceConfig { */ public final Boolean enableVirtualHostStyle; /** - *

Enable write with append so that opendal will send write request with append headers.

+ *

Deprecated: S3 append capability is enabled by default.

+ * + * @deprecated S3 append capability is enabled by default and this option is no longer needed. */ public final Boolean enableWriteWithAppend; /** diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs index 8716ffeccfc2..b0f1c8521f61 100644 --- a/bindings/python/src/services.rs +++ b/bindings/python/src/services.rs @@ -2135,15 +2135,14 @@ submit! { For example, R2 doesn't support stat with `response_content_type` query. disable_write_with_if_match : builtins.bool, optional - Disable write with if match so that opendal will not - send write request with if match headers. - For example, Ceph RADOS S3 doesn't support write - with if matched. + Deprecated: S3 write with If-Match capability is + enabled by default. enable_request_payer : builtins.bool, optional Indicates whether the client agrees to pay for the requests made to the S3 bucket. enable_versioning : builtins.bool, optional - is bucket versioning enabled for this bucket + Deprecated: S3 versioning capability is enabled by + default. enable_virtual_host_style : builtins.bool, optional Enable virtual host style so that opendal will send API requests in virtual host style instead of path @@ -2153,8 +2152,8 @@ submit! { Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` enable_write_with_append : builtins.bool, optional - Enable write with append so that opendal will send - write request with append headers. + Deprecated: S3 append capability is enabled by + default. endpoint : builtins.str, optional endpoint of this backend. Endpoint must be full uri, e.g. @@ -4818,15 +4817,14 @@ submit! { For example, R2 doesn't support stat with `response_content_type` query. disable_write_with_if_match : builtins.bool, optional - Disable write with if match so that opendal will not - send write request with if match headers. - For example, Ceph RADOS S3 doesn't support write - with if matched. + Deprecated: S3 write with If-Match capability is + enabled by default. enable_request_payer : builtins.bool, optional Indicates whether the client agrees to pay for the requests made to the S3 bucket. enable_versioning : builtins.bool, optional - is bucket versioning enabled for this bucket + Deprecated: S3 versioning capability is enabled by + default. enable_virtual_host_style : builtins.bool, optional Enable virtual host style so that opendal will send API requests in virtual host style instead of path @@ -4836,8 +4834,8 @@ submit! { Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` enable_write_with_append : builtins.bool, optional - Enable write with append so that opendal will send - write request with append headers. + Deprecated: S3 append capability is enabled by + default. endpoint : builtins.str, optional endpoint of this backend. Endpoint must be full uri, e.g.