diff --git a/.github/services/s3/0_minio_s3/action.yml b/.github/services/s3/0_minio_s3/action.yml index 2ee8bc8bb314..c2c10eaea174 100644 --- a/.github/services/s3/0_minio_s3/action.yml +++ b/.github/services/s3/0_minio_s3/action.yml @@ -43,4 +43,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/aws_s3/action.yml b/.github/services/s3/aws_s3/action.yml index 13622c405911..a45126876021 100644 --- a/.github/services/s3/aws_s3/action.yml +++ b/.github/services/s3/aws_s3/action.yml @@ -32,3 +32,8 @@ runs: OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key OPENDAL_S3_REGION: op://services/s3/region + + - name: Add capability overrides + shell: bash + run: | + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_list_objects_v1/action.yml b/.github/services/s3/aws_s3_with_list_objects_v1/action.yml index 90bbb961f105..5008dee6f042 100644 --- a/.github/services/s3/aws_s3_with_list_objects_v1/action.yml +++ b/.github/services/s3/aws_s3_with_list_objects_v1/action.yml @@ -37,3 +37,4 @@ runs: shell: bash run: | echo "OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_sse_c/action.yml b/.github/services/s3/aws_s3_with_sse_c/action.yml index 050a17162adf..2dde7218433a 100644 --- a/.github/services/s3/aws_s3_with_sse_c/action.yml +++ b/.github/services/s3/aws_s3_with_sse_c/action.yml @@ -40,4 +40,5 @@ runs: OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256 OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA= OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw== + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/aws_s3_with_versioning/action.yml b/.github/services/s3/aws_s3_with_versioning/action.yml index 01120c5a1bb6..c1f85c397f21 100644 --- a/.github/services/s3/aws_s3_with_versioning/action.yml +++ b/.github/services/s3/aws_s3_with_versioning/action.yml @@ -33,7 +33,7 @@ runs: OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key OPENDAL_S3_REGION: op://services/s3/region - - name: Add extra settings + - name: Add capability overrides shell: bash run: | - echo "OPENDAL_S3_ENABLE_VERSIONING=true" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/aws_s3_with_virtual_host/action.yml b/.github/services/s3/aws_s3_with_virtual_host/action.yml index 08056a146b08..025eff07141b 100644 --- a/.github/services/s3/aws_s3_with_virtual_host/action.yml +++ b/.github/services/s3/aws_s3_with_virtual_host/action.yml @@ -37,3 +37,4 @@ runs: shell: bash run: | echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV + echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV diff --git a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml index 71b550d91e25..e0169e408bab 100644 --- a/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml +++ b/.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml @@ -43,6 +43,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_ENABLE_VERSIONING=true - OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on + OPENDAL_TEST_CAPABILITY_OVERRIDES=write_with_if_match=false,write_can_append=false EOF diff --git a/.github/services/s3/ceph_rados_s3/disable_action.yml b/.github/services/s3/ceph_rados_s3/disable_action.yml index 54e6e7049ddb..ea7d7b45d7cd 100644 --- a/.github/services/s3/ceph_rados_s3/disable_action.yml +++ b/.github/services/s3/ceph_rados_s3/disable_action.yml @@ -41,5 +41,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=demo OPENDAL_S3_SECRET_ACCESS_KEY=demo OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_with_if_match=false,write_can_append=false EOF diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/s3/minio_s3_with_anonymous/action.yml index 3c529f06eeca..0807d2db42fe 100644 --- a/.github/services/s3/minio_s3_with_anonymous/action.yml +++ b/.github/services/s3/minio_s3_with_anonymous/action.yml @@ -49,4 +49,5 @@ runs: OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_ALLOW_ANONYMOUS=on OPENDAL_S3_DISABLE_EC2_METADATA=on + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/minio_s3_with_list_objects_v1/action.yml b/.github/services/s3/minio_s3_with_list_objects_v1/action.yml index 46904d7aaf92..2c8bb3c491a4 100644 --- a/.github/services/s3/minio_s3_with_list_objects_v1/action.yml +++ b/.github/services/s3/minio_s3_with_list_objects_v1/action.yml @@ -44,4 +44,5 @@ runs: OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/.github/services/s3/minio_s3_with_versioning/action.yml b/.github/services/s3/minio_s3_with_versioning/action.yml index e2db993a86f2..1f838a50017f 100644 --- a/.github/services/s3/minio_s3_with_versioning/action.yml +++ b/.github/services/s3/minio_s3_with_versioning/action.yml @@ -44,5 +44,5 @@ runs: OPENDAL_S3_ACCESS_KEY_ID=minioadmin OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_ENABLE_VERSIONING=true + OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false EOF diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/s3/r2/disabled_action.yml index 8a54f78b5c59..fee1b3988f2b 100644 --- a/.github/services/s3/r2/disabled_action.yml +++ b/.github/services/s3/r2/disabled_action.yml @@ -40,4 +40,5 @@ runs: OPENDAL_S3_REGION=auto OPENDAL_S3_DELETE_MAX_SIZE=700 OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true + OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false EOF diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs index 7401a820a0ca..d4415d990877 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs @@ -24,7 +24,9 @@ namespace OpenDAL.Tests; public sealed class BehaviorOperatorFixture : IDisposable { + private const string CapabilityOverridesEnv = "OPENDAL_TEST_CAPABILITY_OVERRIDES"; private readonly Operator? op; + private readonly Capability capability; public string? Scheme { get; } @@ -49,13 +51,20 @@ public BehaviorOperatorFixture() } op = new Operator(scheme, options).WithLayer(new RetryLayer()); + var capabilityOverrides = Environment.GetEnvironmentVariable(CapabilityOverridesEnv); + if (!string.IsNullOrWhiteSpace(capabilityOverrides)) + { + op = op.WithLayer(new CapabilityOverrideLayer(capabilityOverrides)); + } + + capability = op.Info.FullCapability; } public bool IsEnabled => op is not null; public Operator Op => op ?? throw new InvalidOperationException("Behavior operator is not initialized."); - public Capability Capability => Op.Info.FullCapability; + public Capability Capability => capability; public void Dispose() { diff --git a/bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs b/bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs new file mode 100644 index 000000000000..04ab049d87aa --- /dev/null +++ b/bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using OpenDAL.Layer.Abstractions; + +namespace OpenDAL.Layer; + +/// +/// Layer that overrides the full capability exposed by an operator. +/// +public sealed class CapabilityOverrideLayer : ILayer +{ + /// + /// Creates a capability override layer. + /// + /// Comma-separated capability override entries. + public CapabilityOverrideLayer(string overrides) + { + ArgumentException.ThrowIfNullOrWhiteSpace(overrides); + Overrides = overrides; + } + + /// + /// Gets capability override entries. + /// + public string Overrides { get; } + + /// + /// Applies capability overrides to the specified operator. + /// + /// Operator to layer. + /// The layered operator instance. + public Operator Apply(Operator op) + { + ArgumentNullException.ThrowIfNull(op); + ObjectDisposedException.ThrowIf(op.IsInvalid, op); + + var result = NativeMethods.operator_layer_capability_override(op, Overrides); + return op.ApplyLayerResult(result); + } +} diff --git a/bindings/dotnet/OpenDAL/NativeMethods.cs b/bindings/dotnet/OpenDAL/NativeMethods.cs index 609408fedc26..fd8ecb3ad41a 100644 --- a/bindings/dotnet/OpenDAL/NativeMethods.cs +++ b/bindings/dotnet/OpenDAL/NativeMethods.cs @@ -154,6 +154,13 @@ internal static partial OpenDALOperatorResult operator_layer_concurrent_limit( nuint permits ); + [LibraryImport(__DllName, EntryPoint = "operator_layer_capability_override", StringMarshalling = StringMarshalling.Utf8)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial OpenDALOperatorResult operator_layer_capability_override( + Operator op, + string overrides + ); + [LibraryImport(__DllName, EntryPoint = "operator_layer_timeout")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial OpenDALOperatorResult operator_layer_timeout( diff --git a/bindings/dotnet/src/operator.rs b/bindings/dotnet/src/operator.rs index af725a3a23cc..a66f6b9f9581 100644 --- a/bindings/dotnet/src/operator.rs +++ b/bindings/dotnet/src/operator.rs @@ -410,6 +410,37 @@ fn operator_layer_concurrent_limit_inner( Ok(Box::into_raw(Box::new(op.clone().layer(concurrent_limit))) as *mut c_void) } +/// Create a new operator layered with capability override behavior. +/// +/// The current operator is not modified. Returned pointer must be released with +/// `operator_free`. +/// # Safety +/// +/// - `op` must be a valid operator pointer from `operator_construct`. +/// - `overrides` must be a valid null-terminated UTF-8 string. +#[unsafe(no_mangle)] +pub extern "C" fn operator_layer_capability_override( + op: *const opendal::Operator, + overrides: *const c_char, +) -> OpendalOperatorResult { + match operator_layer_capability_override_inner(op, overrides) { + Ok(value) => OpendalOperatorResult::ok(value), + Err(error) => OpendalOperatorResult::from_error(error), + } +} + +fn operator_layer_capability_override_inner( + op: *const opendal::Operator, + overrides: *const c_char, +) -> Result<*mut c_void, OpenDALError> { + let op = require_operator(op)?; + let overrides = require_cstr(overrides, "capability overrides")?; + let layer = opendal::layers::CapabilityOverrideLayer::from_overrides(overrides) + .map_err(OpenDALError::from_opendal_error)?; + + Ok(Box::into_raw(Box::new(op.clone().layer(layer))) as *mut c_void) +} + /// Create a new operator layered with timeout behavior. /// /// The current operator is not modified. Returned pointer must be released with diff --git a/bindings/java/src/layer.rs b/bindings/java/src/layer.rs index c649bfa0286b..5f41567c8c9a 100644 --- a/bindings/java/src/layer.rs +++ b/bindings/java/src/layer.rs @@ -17,12 +17,16 @@ use std::time::Duration; +use crate::Result; +use crate::convert::jstring_to_string; use jni::JNIEnv; use jni::objects::JClass; +use jni::objects::JString; use jni::sys::jboolean; use jni::sys::jfloat; use jni::sys::jlong; use opendal::Operator; +use opendal::layers::CapabilityOverrideLayer; use opendal::layers::ConcurrentLimitLayer; use opendal::layers::RetryLayer; @@ -51,6 +55,30 @@ pub extern "system" fn Java_org_apache_opendal_layer_RetryLayer_doLayer( Box::into_raw(Box::new(op.clone().layer(retry))) as jlong } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_apache_opendal_layer_CapabilityOverrideLayer_doLayer( + mut env: JNIEnv, + _: JClass, + op: *mut Operator, + overrides: JString, +) -> jlong { + intern_capability_override_layer(&mut env, op, overrides).unwrap_or_else(|e| { + e.throw(&mut env); + 0 + }) +} + +fn intern_capability_override_layer( + env: &mut JNIEnv, + op: *mut Operator, + overrides: JString, +) -> Result { + let op = unsafe { &*op }; + let overrides = jstring_to_string(env, &overrides)?; + let layer = CapabilityOverrideLayer::from_overrides(&overrides)?; + Ok(Box::into_raw(Box::new(op.clone().layer(layer))) as jlong) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_org_apache_opendal_layer_ConcurrentLimitLayer_doLayer( _: JNIEnv, diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java index 8dfbe8479c7d..d08e9300a89a 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java +++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java @@ -2917,8 +2917,9 @@ class S3 implements ServiceConfig { */ public final Boolean disableStatWithOverride; /** - *

Disable write with if match so that opendal will not send write request with if match headers.

- *

For example, Ceph RADOS S3 doesn't support write with if matched.

+ *

Deprecated: S3 write with If-Match capability is enabled by default.

+ * + * @deprecated S3 write with If-Match capability is enabled by default and this option is no longer needed. */ public final Boolean disableWriteWithIfMatch; /** @@ -2926,7 +2927,9 @@ class S3 implements ServiceConfig { */ public final Boolean enableRequestPayer; /** - *

is bucket versioning enabled for this bucket

+ *

Deprecated: S3 versioning capability is enabled by default.

+ * + * @deprecated S3 versioning capability is enabled by default and this option is no longer needed. */ public final Boolean enableVersioning; /** @@ -2939,7 +2942,9 @@ class S3 implements ServiceConfig { */ public final Boolean enableVirtualHostStyle; /** - *

Enable write with append so that opendal will send write request with append headers.

+ *

Deprecated: S3 append capability is enabled by default.

+ * + * @deprecated S3 append capability is enabled by default and this option is no longer needed. */ public final Boolean enableWriteWithAppend; /** diff --git a/bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java b/bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java new file mode 100644 index 000000000000..ace54af04028 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.layer; + +import org.apache.opendal.Layer; + +/** + * Layer that overrides the full capability exposed by an operator. + */ +public class CapabilityOverrideLayer extends Layer { + + private final String overrides; + + public CapabilityOverrideLayer(String overrides) { + this.overrides = overrides; + } + + @Override + protected long layer(long nativeOp) { + return doLayer(nativeOp, overrides); + } + + private static native long doLayer(long nativeHandle, String overrides); +} diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java index 367e79c66715..3ad216a6acf9 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java @@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.opendal.AsyncOperator; import org.apache.opendal.Operator; +import org.apache.opendal.layer.CapabilityOverrideLayer; import org.apache.opendal.layer.RetryLayer; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; @@ -67,6 +68,10 @@ public void beforeAll(ExtensionContext context) { @Cleanup final AsyncOperator op = AsyncOperator.of(scheme.toLowerCase().replace('_', '-'), config); this.asyncOperator = op.layer(RetryLayer.builder().build()); + final String capabilityOverrides = dotenv.get("OPENDAL_TEST_CAPABILITY_OVERRIDES"); + if (capabilityOverrides != null && !capabilityOverrides.trim().isEmpty()) { + this.asyncOperator = this.asyncOperator.layer(new CapabilityOverrideLayer(capabilityOverrides)); + } this.operator = this.asyncOperator.blocking(); this.scheme = scheme; diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts index 999bb81471b7..8ac5619d1bac 100644 --- a/bindings/nodejs/generated.d.ts +++ b/bindings/nodejs/generated.d.ts @@ -207,6 +207,17 @@ export declare class Capability { get shared(): boolean } +/** + * Capability override layer + * + * Override the full capability exposed by an operator. + */ +export declare class CapabilityOverrideLayer { + /** Create a new CapabilityOverrideLayer from capability override entries. */ + constructor(overrides: string) + build(): ExternalObject +} + /** * Concurrent limit layer * diff --git a/bindings/nodejs/generated.js b/bindings/nodejs/generated.js index dc74d0e045b6..612c1dfa0193 100644 --- a/bindings/nodejs/generated.js +++ b/bindings/nodejs/generated.js @@ -531,6 +531,7 @@ module.exports.BlockingLister = nativeBinding.BlockingLister module.exports.BlockingReader = nativeBinding.BlockingReader module.exports.BlockingWriter = nativeBinding.BlockingWriter module.exports.Capability = nativeBinding.Capability +module.exports.CapabilityOverrideLayer = nativeBinding.CapabilityOverrideLayer module.exports.ConcurrentLimitLayer = nativeBinding.ConcurrentLimitLayer module.exports.Entry = nativeBinding.Entry module.exports.Layer = nativeBinding.Layer diff --git a/bindings/nodejs/index.mjs b/bindings/nodejs/index.mjs index 1eac4ec77dca..59b0ee47eede 100644 --- a/bindings/nodejs/index.mjs +++ b/bindings/nodejs/index.mjs @@ -120,7 +120,16 @@ class BlockingWriteStream extends Writable { } import * as generated from './generated.js' -const { Operator, RetryLayer, ConcurrentLimitLayer, BlockingReader, Reader, BlockingWriter, Writer } = generated +const { + Operator, + CapabilityOverrideLayer, + RetryLayer, + ConcurrentLimitLayer, + BlockingReader, + Reader, + BlockingWriter, + Writer, +} = generated BlockingReader.prototype.createReadStream = function (options) { return new BlockingReadStream(this, options) @@ -139,6 +148,7 @@ Writer.prototype.createWriteStream = function (options) { } export const layers = { + CapabilityOverrideLayer, RetryLayer, ConcurrentLimitLayer, } diff --git a/bindings/nodejs/src/layer.rs b/bindings/nodejs/src/layer.rs index ff6e9dd2bb81..b78e458b7193 100644 --- a/bindings/nodejs/src/layer.rs +++ b/bindings/nodejs/src/layer.rs @@ -18,6 +18,7 @@ use std::time::Duration; use napi::bindgen_prelude::External; +use napi::bindgen_prelude::Result; pub trait NodeLayer: Send + Sync { fn layer(&self, op: opendal::Operator) -> opendal::Operator; @@ -35,6 +36,37 @@ impl NodeLayer for opendal::layers::RetryLayer { } } +impl NodeLayer for opendal::layers::CapabilityOverrideLayer { + fn layer(&self, op: opendal::Operator) -> opendal::Operator { + op.layer(self.clone()) + } +} + +/// Capability override layer +/// +/// Override the full capability exposed by an operator. +#[napi] +pub struct CapabilityOverrideLayer { + overrides: String, +} + +#[napi] +impl CapabilityOverrideLayer { + /// Create a new CapabilityOverrideLayer from capability override entries. + #[napi(constructor)] + pub fn new(overrides: String) -> Self { + Self { overrides } + } + + #[napi] + pub fn build(&self) -> Result> { + let l = opendal::layers::CapabilityOverrideLayer::from_overrides(&self.overrides) + .map_err(|err| napi::Error::from_reason(format!("{err}")))?; + + Ok(External::new(Layer { inner: Box::new(l) })) + } +} + /// Retry layer /// /// Add retry for temporary failed operations. diff --git a/bindings/nodejs/tests/suites/index.mjs b/bindings/nodejs/tests/suites/index.mjs index 36d7de093698..36794e62c683 100644 --- a/bindings/nodejs/tests/suites/index.mjs +++ b/bindings/nodejs/tests/suites/index.mjs @@ -61,6 +61,9 @@ export function runner(testName, scheme) { retryLayer.maxTimes = 4 operator = operator.layer(retryLayer.build()) + if (process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES) { + operator = operator.layer(new layers.CapabilityOverrideLayer(process.env.OPENDAL_TEST_CAPABILITY_OVERRIDES).build()) + } describe.skipIf(!operator)(testName, () => { AsyncIOTestRun(operator) diff --git a/bindings/python/python/opendal/layers.pyi b/bindings/python/python/opendal/layers.pyi index d46e6f5e18eb..48862d1f513f 100644 --- a/bindings/python/python/opendal/layers.pyi +++ b/bindings/python/python/opendal/layers.pyi @@ -21,6 +21,24 @@ import builtins import typing +@typing.final +class CapabilityOverrideLayer(Layer): + r"""A layer that overrides the full capability exposed by an operator.""" + + def __new__(cls, overrides: builtins.str) -> CapabilityOverrideLayer: + r""" + Create a new CapabilityOverrideLayer from capability override entries. + + Parameters + ---------- + overrides : str + Comma-separated capability override entries. + + Returns + ------- + CapabilityOverrideLayer + """ + @typing.final class ConcurrentLimitLayer(Layer): r""" diff --git a/bindings/python/src/layers.rs b/bindings/python/src/layers.rs index 84365f88e530..92957e7ec2f8 100644 --- a/bindings/python/src/layers.rs +++ b/bindings/python/src/layers.rs @@ -29,6 +29,45 @@ pub trait PythonLayer: Send + Sync { #[pyclass(module = "opendal.layers", subclass)] pub struct Layer(pub Box); +/// A layer that overrides the full capability exposed by an operator. +#[gen_stub_pyclass] +#[pyclass(module = "opendal.layers", extends=Layer)] +#[derive(Clone)] +pub struct CapabilityOverrideLayer(ocore::layers::CapabilityOverrideLayer); + +impl PythonLayer for CapabilityOverrideLayer { + fn layer(&self, op: Operator) -> Operator { + op.layer(self.0.clone()) + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl CapabilityOverrideLayer { + /// Create a new CapabilityOverrideLayer from capability override entries. + /// + /// Parameters + /// ---------- + /// overrides : str + /// Comma-separated capability override entries. + /// + /// Returns + /// ------- + /// CapabilityOverrideLayer + #[gen_stub(override_return_type(type_repr = "CapabilityOverrideLayer"))] + #[new] + #[pyo3(signature = (overrides))] + fn new(overrides: &str) -> PyResult> { + let layer = Self( + ocore::layers::CapabilityOverrideLayer::from_overrides(overrides) + .map_err(format_pyerr)?, + ); + let class = PyClassInitializer::from(Layer(Box::new(layer.clone()))).add_subclass(layer); + + Ok(class) + } +} + /// A layer that retries operations that fail with temporary errors. /// /// Operations are retried if they fail with an error for which diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index bf413eb7501f..7f9a22148693 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -64,7 +64,13 @@ fn _opendal(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { py, m, "layers", - [Layer, RetryLayer, ConcurrentLimitLayer, MimeGuessLayer] + [ + Layer, + CapabilityOverrideLayer, + RetryLayer, + ConcurrentLimitLayer, + MimeGuessLayer + ] )?; // Types module diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs index c369257c8d2c..c29b2e1d4f00 100644 --- a/bindings/python/src/services.rs +++ b/bindings/python/src/services.rs @@ -2139,15 +2139,14 @@ submit! { For example, R2 doesn't support stat with `response_content_type` query. disable_write_with_if_match : builtins.bool, optional - Disable write with if match so that opendal will not - send write request with if match headers. - For example, Ceph RADOS S3 doesn't support write - with if matched. + Deprecated: S3 write with If-Match capability is + enabled by default. enable_request_payer : builtins.bool, optional Indicates whether the client agrees to pay for the requests made to the S3 bucket. enable_versioning : builtins.bool, optional - is bucket versioning enabled for this bucket + Deprecated: S3 versioning capability is enabled by + default. enable_virtual_host_style : builtins.bool, optional Enable virtual host style so that opendal will send API requests in virtual host style instead of path @@ -2157,8 +2156,8 @@ submit! { Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` enable_write_with_append : builtins.bool, optional - Enable write with append so that opendal will send - write request with append headers. + Deprecated: S3 append capability is enabled by + default. endpoint : builtins.str, optional endpoint of this backend. Endpoint must be full uri, e.g. @@ -4826,15 +4825,14 @@ submit! { For example, R2 doesn't support stat with `response_content_type` query. disable_write_with_if_match : builtins.bool, optional - Disable write with if match so that opendal will not - send write request with if match headers. - For example, Ceph RADOS S3 doesn't support write - with if matched. + Deprecated: S3 write with If-Match capability is + enabled by default. enable_request_payer : builtins.bool, optional Indicates whether the client agrees to pay for the requests made to the S3 bucket. enable_versioning : builtins.bool, optional - is bucket versioning enabled for this bucket + Deprecated: S3 versioning capability is enabled by + default. enable_virtual_host_style : builtins.bool, optional Enable virtual host style so that opendal will send API requests in virtual host style instead of path @@ -4844,8 +4842,8 @@ submit! { Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com` enable_write_with_append : builtins.bool, optional - Enable write with append so that opendal will send - write request with append headers. + Deprecated: S3 append capability is enabled by + default. endpoint : builtins.str, optional endpoint of this backend. Endpoint must be full uri, e.g. diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index 5bf32d54588c..3c89efcc2d9f 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -25,6 +25,7 @@ load_dotenv() pytest_plugins = ("pytest_asyncio",) +CAPABILITY_OVERRIDES_ENV = "OPENDAL_TEST_CAPABILITY_OVERRIDES" def pytest_configure(config): @@ -59,12 +60,16 @@ def setup_config(service_name): @pytest.fixture(scope="session") def async_operator(service_name, setup_config): - return ( + operator = ( opendal.AsyncOperator(service_name, **setup_config) .layer(opendal.layers.RetryLayer()) .layer(opendal.layers.ConcurrentLimitLayer(1024)) .layer(opendal.layers.MimeGuessLayer()) ) + overrides = os.environ.get(CAPABILITY_OVERRIDES_ENV) + if overrides: + operator = operator.layer(opendal.layers.CapabilityOverrideLayer(overrides)) + return operator @pytest.fixture(scope="session") diff --git a/core/core/src/layers/capability_override.rs b/core/core/src/layers/capability_override.rs new file mode 100644 index 000000000000..5f3dd97cb1d3 --- /dev/null +++ b/core/core/src/layers/capability_override.rs @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; +use std::sync::Arc; + +use serde_json::Map; +use serde_json::Number; +use serde_json::Value; + +use crate::raw::*; +use crate::*; + +/// Layer for overriding an accessor's full capability. +/// +/// This layer updates [`Capability`] exposed by +/// [`OperatorInfo::full_capability`][crate::OperatorInfo::full_capability] +/// without changing the accessor's native capability. It is useful when the +/// backend implementation supports a capability, but the specific endpoint or +/// test setup needs to disable or tune it. +/// +/// # Examples +/// +/// ```no_run +/// use opendal_core::layers::CapabilityOverrideLayer; +/// use opendal_core::services; +/// use opendal_core::Operator; +/// use opendal_core::Result; +/// +/// # fn main() -> Result<()> { +/// let op = Operator::new(services::Memory::default())? +/// .layer(CapabilityOverrideLayer::new(|mut cap| { +/// cap.write_with_if_match = false; +/// cap.delete_max_size = Some(700); +/// cap +/// })) +/// .finish(); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone)] +pub struct CapabilityOverrideLayer { + apply: Arc Capability + Send + Sync>, +} + +impl fmt::Debug for CapabilityOverrideLayer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CapabilityOverrideLayer") + .finish_non_exhaustive() + } +} + +impl CapabilityOverrideLayer { + /// Create a new [`CapabilityOverrideLayer`]. + pub fn new(apply: impl Fn(Capability) -> Capability + Send + Sync + 'static) -> Self { + Self { + apply: Arc::new(apply), + } + } + + /// Create a new [`CapabilityOverrideLayer`] from capability override entries. + /// + /// The input is a comma-separated list of capability assignments: + /// + /// - `read=true` sets a boolean capability to `true` + /// - `read=false` sets a boolean capability to `false` + /// - `delete_max_size=1000` sets a numeric capability + /// - `delete_max_size=none` unsets an optional capability + pub fn from_overrides(input: &str) -> Result { + let overrides = CapabilityOverrides::parse(input)?; + Ok(Self::new(move |cap| overrides.apply(cap))) + } +} + +impl Layer for CapabilityOverrideLayer { + type LayeredAccess = A; + + fn layer(&self, inner: A) -> Self::LayeredAccess { + let info = inner.info(); + let apply = self.apply.clone(); + info.update_full_capability(|cap| apply(cap)); + inner + } +} + +#[derive(Clone, Debug, Default)] +struct CapabilityOverrides { + values: Map, +} + +impl CapabilityOverrides { + fn parse(input: &str) -> Result { + let mut overrides = Self::default(); + + for token in input.split(',').map(str::trim).filter(|v| !v.is_empty()) { + let (name, value) = parse_capability_override(token)?; + overrides.values.insert(name.to_string(), value); + overrides.try_apply(Capability::default()).map_err(|err| { + invalid_capability_override(token, &format!("failed to apply override: {err}")) + })?; + } + + Ok(overrides) + } + + fn apply(&self, cap: Capability) -> Capability { + self.try_apply(cap) + .expect("capability overrides must be validated before applying") + } + + fn try_apply(&self, cap: Capability) -> Result { + let mut value = serde_json::to_value(cap).map_err(|err| { + Error::new( + ErrorKind::Unexpected, + format!("failed to serialize capability: {err}"), + ) + })?; + let object = value.as_object_mut().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "serialized capability must be a JSON object", + ) + })?; + object.extend(self.values.clone()); + + serde_json::from_value(value).map_err(|err| { + Error::new( + ErrorKind::ConfigInvalid, + format!("failed to deserialize capability overrides: {err}"), + ) + }) + } +} + +fn parse_capability_override(token: &str) -> Result<(&str, Value)> { + let Some((name, value)) = token.split_once('=') else { + return Err(invalid_capability_override( + token, + "expected `capability=value`", + )); + }; + + Ok(( + name.trim(), + parse_capability_value(value.trim()) + .map_err(|err| invalid_capability_override(token, &err.to_string()))?, + )) +} + +fn invalid_capability_override(token: &str, reason: &str) -> Error { + Error::new( + ErrorKind::ConfigInvalid, + format!("invalid capability override entry `{token}`: {reason}"), + ) +} + +fn parse_capability_value(value: &str) -> Result { + match value { + "true" | "on" | "yes" => Ok(Value::Bool(true)), + "false" | "off" | "no" => Ok(Value::Bool(false)), + "none" | "null" | "unset" => Ok(Value::Null), + _ => value + .parse::() + .map(|v| Value::Number(Number::from(v))) + .map_err(|_| { + Error::new( + ErrorKind::ConfigInvalid, + "expected a boolean, non-negative integer, or `none`", + ) + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Operator; + use crate::services; + + #[test] + fn capability_override_updates_full_capability_only() -> Result<()> { + let op = Operator::new(services::Memory::default())? + .layer(CapabilityOverrideLayer::new(|mut cap| { + cap.read = false; + cap.delete_max_size = Some(7); + cap + })) + .finish(); + + assert!(!op.info().full_capability().read); + assert_eq!(op.info().full_capability().delete_max_size, Some(7)); + + assert!(op.info().native_capability().read); + assert_ne!( + op.info().native_capability().delete_max_size, + op.info().full_capability().delete_max_size + ); + + Ok(()) + } + + #[test] + fn parse_capability_overrides() -> Result<()> { + let layer = CapabilityOverrideLayer::from_overrides( + "read=false,write_can_append=true,delete_max_size=7", + )?; + let op = Operator::new(services::Memory::default())? + .layer(layer) + .finish(); + + assert!(!op.info().full_capability().read); + assert!(op.info().full_capability().write_can_append); + assert_eq!(op.info().full_capability().delete_max_size, Some(7)); + + Ok(()) + } + + #[test] + fn parse_bool_assignments_and_unset_sizes() -> Result<()> { + let layer = + CapabilityOverrideLayer::from_overrides("read=false,write=true,delete_max_size=none")?; + let op = Operator::new(services::Memory::default())? + .layer(layer) + .finish(); + + assert!(!op.info().full_capability().read); + assert!(op.info().full_capability().write); + assert_eq!(op.info().full_capability().delete_max_size, None); + + Ok(()) + } + + #[test] + fn reject_unknown_capability() { + let err = CapabilityOverrideLayer::from_overrides("not_a_capability=false").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reject_capability_shorthand() { + let err = CapabilityOverrideLayer::from_overrides("-read").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reject_invalid_capability_type() { + let err = CapabilityOverrideLayer::from_overrides("read=1").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } +} diff --git a/core/core/src/layers/mod.rs b/core/core/src/layers/mod.rs index 3a1f45bf7654..f81b8c837cdc 100644 --- a/core/core/src/layers/mod.rs +++ b/core/core/src/layers/mod.rs @@ -29,6 +29,9 @@ pub(crate) use complete::CompleteLayer; mod simulate; pub use simulate::SimulateLayer; +mod capability_override; +pub use capability_override::CapabilityOverrideLayer; + mod correctness_check; pub(crate) use correctness_check::CorrectnessCheckLayer; diff --git a/core/core/src/types/capability.rs b/core/core/src/types/capability.rs index 7c49eee181b6..e883d316b5ee 100644 --- a/core/core/src/types/capability.rs +++ b/core/core/src/types/capability.rs @@ -17,6 +17,9 @@ use std::fmt::Debug; +use serde::Deserialize; +use serde::Serialize; + /// Capability defines the supported operations and their constraints for a storage Operator. /// /// # Overview @@ -62,7 +65,8 @@ use std::fmt::Debug; /// - Metadata Results: Returning metadata capabilities (e.g., `stat_has_content_length`) /// /// All capability fields are public and can be accessed directly. -#[derive(Copy, Clone, Default)] +#[derive(Copy, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Capability { /// Indicates if the operator supports metadata retrieval operations. pub stat: bool, diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index c998267d2180..696d5835a485 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -454,10 +454,12 @@ impl S3Builder { self } - /// Set bucket versioning status for this backend - pub fn enable_versioning(mut self, enabled: bool) -> Self { - self.config.enable_versioning = enabled; - + /// Deprecated: S3 versioning capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 versioning capability is enabled by default and this option is no longer needed." + )] + pub fn enable_versioning(self, _enabled: bool) -> Self { self } @@ -561,15 +563,21 @@ impl S3Builder { self } - /// Disable write with if match so that opendal will not send write request with if match headers. - pub fn disable_write_with_if_match(mut self) -> Self { - self.config.disable_write_with_if_match = true; + /// Deprecated: S3 write with If-Match capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 write with If-Match capability is enabled by default and this option is no longer needed." + )] + pub fn disable_write_with_if_match(self) -> Self { self } - /// Enable write with append so that opendal will send write request with append headers. - pub fn enable_write_with_append(mut self) -> Self { - self.config.enable_write_with_append = true; + /// Deprecated: S3 append capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 append capability is enabled by default and this option is no longer needed." + )] + pub fn enable_write_with_append(self) -> Self { self } @@ -905,7 +913,7 @@ impl Builder for S3Builder { stat_with_override_content_disposition: !config .disable_stat_with_override, stat_with_override_content_type: !config.disable_stat_with_override, - stat_with_version: config.enable_versioning, + stat_with_version: true, read: true, read_with_if_match: true, @@ -915,18 +923,18 @@ impl Builder for S3Builder { read_with_override_cache_control: true, read_with_override_content_disposition: true, read_with_override_content_type: true, - read_with_version: config.enable_versioning, + read_with_version: true, write: true, write_can_empty: true, write_can_multi: true, - write_can_append: config.enable_write_with_append, + write_can_append: true, write_with_cache_control: true, write_with_content_type: true, write_with_content_disposition: true, write_with_content_encoding: true, - write_with_if_match: !config.disable_write_with_if_match, + write_with_if_match: true, write_with_if_not_exists: true, write_with_user_metadata: true, @@ -945,7 +953,7 @@ impl Builder for S3Builder { delete: true, delete_max_size: Some(delete_max_size), - delete_with_version: config.enable_versioning, + delete_with_version: true, copy: true, @@ -953,8 +961,8 @@ impl Builder for S3Builder { list_with_limit: true, list_with_start_after: true, list_with_recursive: true, - list_with_versions: config.enable_versioning, - list_with_deleted: config.enable_versioning, + list_with_versions: true, + list_with_deleted: true, presign: true, presign_stat: true, diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs index 658af8b13992..d47233e1e0d9 100644 --- a/core/services/s3/src/config.rs +++ b/core/services/s3/src/config.rs @@ -43,7 +43,11 @@ pub struct S3Config { /// required. #[serde(alias = "aws_bucket", alias = "aws_bucket_name", alias = "bucket_name")] pub bucket: String, - /// is bucket versioning enabled for this bucket + /// Deprecated: S3 versioning capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 versioning capability is enabled by default and this option is no longer needed." + )] pub enable_versioning: bool, /// endpoint of this backend. /// @@ -210,12 +214,18 @@ pub struct S3Config { /// - "md5" #[serde(alias = "aws_checksum_algorithm")] pub checksum_algorithm: Option, - /// Disable write with if match so that opendal will not send write request with if match headers. - /// - /// For example, Ceph RADOS S3 doesn't support write with if matched. + /// Deprecated: S3 write with If-Match capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 write with If-Match capability is enabled by default and this option is no longer needed." + )] pub disable_write_with_if_match: bool, - /// Enable write with append so that opendal will send write request with append headers. + /// Deprecated: S3 append capability is enabled by default. + #[deprecated( + since = "0.57.0", + note = "S3 append capability is enabled by default and this option is no longer needed." + )] pub enable_write_with_append: bool, /// OpenDAL uses List Objects V2 by default to list objects. diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md index df279f35b7ec..8b3fc6024c2d 100644 --- a/core/services/s3/src/docs.md +++ b/core/services/s3/src/docs.md @@ -29,8 +29,10 @@ This service can be used to: - `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for backend. - `disable_config_load`: Disable aws config load from env. - `enable_virtual_host_style`: Enable virtual host style. -- `disable_write_with_if_match`: Disable write with if match. - `enable_request_payer`: Enable the request payer for backend. +- `disable_write_with_if_match`: Deprecated. S3 write with If-Match capability is enabled by default and this option is no longer needed. +- `enable_versioning`: Deprecated. S3 versioning capability is enabled by default and this option is no longer needed. +- `enable_write_with_append`: Deprecated. S3 append capability is enabled by default and this option is no longer needed. - `default_acl`: Define the default access control list (ACL) when creating a new object. Note that some s3 services like minio do not support this option. Refer to [`S3Builder`]'s public API docs for more information. diff --git a/core/testkit/src/utils.rs b/core/testkit/src/utils.rs index e356ba038046..3d53ee4cbeff 100644 --- a/core/testkit/src/utils.rs +++ b/core/testkit/src/utils.rs @@ -21,12 +21,15 @@ use std::sync::LazyLock; use opendal_core::Operator; use opendal_core::Result; +use opendal_core::layers::CapabilityOverrideLayer; use opendal_layer_logging::LoggingLayer; use opendal_layer_retry::RetryLayer; use opendal_layer_timeout::TimeoutLayer; use sha2::Digest; use sha2::Sha256; +const OPENDAL_TEST_CAPABILITY_OVERRIDES: &str = "OPENDAL_TEST_CAPABILITY_OVERRIDES"; + pub(crate) fn sha256_digest(data: impl AsRef<[u8]>) -> String { use std::fmt::Write; @@ -86,7 +89,11 @@ pub fn init_test_service() -> Result> { // string-based scheme uses a hyphen ('-') as the connector let scheme = scheme.replace('_', "-"); - let op = Operator::via_iter(scheme, cfg).expect("must succeed"); + let mut op = Operator::via_iter(scheme, cfg).expect("must succeed"); + + if let Ok(overrides) = env::var(OPENDAL_TEST_CAPABILITY_OVERRIDES) { + op = op.layer(CapabilityOverrideLayer::from_overrides(&overrides)?); + } let op = op .layer(LoggingLayer::default()) diff --git a/core/tests/behavior/README.md b/core/tests/behavior/README.md index 186c6c030801..5eb36d56a748 100644 --- a/core/tests/behavior/README.md +++ b/core/tests/behavior/README.md @@ -32,6 +32,16 @@ OPENDAL_FS_ROOT=/tmp/ Notice: If the env variables are not set, all behavior tests will be skipped by default. +### Override Capability + +Behavior tests are selected by the operator's full capability. Test setups can override capability with `OPENDAL_TEST_CAPABILITY_OVERRIDES`: + +```shell +OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_max_size=700 +``` + +Use `capability=false` or `capability=true` to set boolean capabilities, and `capability=value` to set numeric capability values. + ## Run Use `OPENDAL_TEST` to control which service to test: diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs index 90b5bdecf19a..3d1893a7cd46 100644 --- a/core/tests/behavior/async_delete.rs +++ b/core/tests/behavior/async_delete.rs @@ -17,7 +17,7 @@ use anyhow::Result; use futures::TryStreamExt; -use opendal::raw::Access; +use opendal::layers::CapabilityOverrideLayer; use opendal::raw::OpDelete; use crate::*; @@ -351,7 +351,7 @@ pub async fn test_batch_delete(op: Operator) -> Result<()> { } cap.delete_max_size = Some(2); - op.inner().info().update_full_capability(|_| cap); + let op = op.layer(CapabilityOverrideLayer::new(move |_| cap)); let mut files = Vec::new(); for _ in 0..5 { @@ -385,7 +385,7 @@ pub async fn test_batch_delete_with_version(op: Operator) -> Result<()> { } cap.delete_max_size = Some(2); - op.inner().info().update_full_capability(|_| cap); + let op = op.layer(CapabilityOverrideLayer::new(move |_| cap)); let mut files = Vec::new(); for _ in 0..5 {