feat(core): add capability override layer

.github/services/s3/0_minio_s3/action.yml

@@ -43,4 +43,5 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID=minioadmin
         OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
         OPENDAL_S3_REGION=us-east-1
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
         EOF

.github/services/s3/aws_s3/action.yml

@@ -32,3 +32,8 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID: op://services/s3/access_key_id
         OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
         OPENDAL_S3_REGION: op://services/s3/region
+
+    - name: Add capability overrides
+      shell: bash
+      run: |
+        echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV

.github/services/s3/aws_s3_with_list_objects_v1/action.yml

@@ -37,3 +37,4 @@ runs:
       shell: bash
       run: |
         echo "OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true" >> $GITHUB_ENV
+        echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV

.github/services/s3/aws_s3_with_sse_c/action.yml

@@ -40,4 +40,5 @@ runs:
         OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM=AES256
         OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY=MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=
         OPENDAL_S3_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5=zZ5FnqcIqUjVwvWmyog4zw==
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
         EOF

.github/services/s3/aws_s3_with_versioning/action.yml

@@ -33,7 +33,7 @@ runs:
         OPENDAL_S3_SECRET_ACCESS_KEY: op://services/s3/secret_access_key
         OPENDAL_S3_REGION: op://services/s3/region
 
-    - name: Add extra settings
+    - name: Add capability overrides
       shell: bash
       run: |
-        echo "OPENDAL_S3_ENABLE_VERSIONING=true" >> $GITHUB_ENV
+        echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false" >> $GITHUB_ENV

.github/services/s3/aws_s3_with_virtual_host/action.yml

@@ -37,3 +37,4 @@ runs:
       shell: bash
       run: |
         echo "OPENDAL_S3_ENABLE_VIRTUAL_HOST_STYLE=on" >> $GITHUB_ENV
+        echo "OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false" >> $GITHUB_ENV

.github/services/s3/ceph_radios_s3_with_versioning/disable_action.yml

@@ -43,6 +43,5 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID=demo
         OPENDAL_S3_SECRET_ACCESS_KEY=demo
         OPENDAL_S3_REGION=us-east-1
-        OPENDAL_S3_ENABLE_VERSIONING=true
-        OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_with_if_match=false,write_can_append=false
         EOF

.github/services/s3/ceph_rados_s3/disable_action.yml

@@ -41,5 +41,5 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID=demo
         OPENDAL_S3_SECRET_ACCESS_KEY=demo
         OPENDAL_S3_REGION=us-east-1
-        OPENDAL_S3_DISABLE_WRITE_WITH_IF_MATCH=on
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_with_if_match=false,write_can_append=false
         EOF

.github/services/s3/minio_s3_with_anonymous/action.yml

@@ -49,4 +49,5 @@ runs:
         OPENDAL_S3_REGION=us-east-1
         OPENDAL_S3_ALLOW_ANONYMOUS=on
         OPENDAL_S3_DISABLE_EC2_METADATA=on
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
         EOF

.github/services/s3/minio_s3_with_list_objects_v1/action.yml

@@ -44,4 +44,5 @@ runs:
         OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
         OPENDAL_S3_REGION=us-east-1
         OPENDAL_S3_DISABLE_LIST_OBJECTS_V2=true
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
         EOF

.github/services/s3/minio_s3_with_versioning/action.yml

@@ -44,5 +44,5 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID=minioadmin
         OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
         OPENDAL_S3_REGION=us-east-1
-        OPENDAL_S3_ENABLE_VERSIONING=true
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
         EOF

.github/services/s3/r2/disabled_action.yml

@@ -40,4 +40,5 @@ runs:
         OPENDAL_S3_REGION=auto
         OPENDAL_S3_DELETE_MAX_SIZE=700
         OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
         EOF

bindings/dotnet/OpenDAL.Tests/Behavior/BehaviorOperatorFixture.cs

@@ -24,7 +24,9 @@ namespace OpenDAL.Tests;
 
 public sealed class BehaviorOperatorFixture : IDisposable
 {
+    private const string CapabilityOverridesEnv = "OPENDAL_TEST_CAPABILITY_OVERRIDES";
     private readonly Operator? op;
+    private readonly Capability capability;
 
     public string? Scheme { get; }
 
@@ -49,13 +51,20 @@ public BehaviorOperatorFixture()
         }
 
         op = new Operator(scheme, options).WithLayer(new RetryLayer());
+        var capabilityOverrides = Environment.GetEnvironmentVariable(CapabilityOverridesEnv);
+        if (!string.IsNullOrWhiteSpace(capabilityOverrides))
+        {
+            op = op.WithLayer(new CapabilityOverrideLayer(capabilityOverrides));
+        }
+
+        capability = op.Info.FullCapability;
     }
 
     public bool IsEnabled => op is not null;
 
     public Operator Op => op ?? throw new InvalidOperationException("Behavior operator is not initialized.");
 
-    public Capability Capability => Op.Info.FullCapability;
+    public Capability Capability => capability;
 
     public void Dispose()
     {

bindings/dotnet/OpenDAL/Layer/CapabilityOverrideLayer.cs

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using OpenDAL.Layer.Abstractions;
+
+namespace OpenDAL.Layer;
+
+/// <summary>
+/// Layer that overrides the full capability exposed by an operator.
+/// </summary>
+public sealed class CapabilityOverrideLayer : ILayer
+{
+    /// <summary>
+    /// Creates a capability override layer.
+    /// </summary>
+    /// <param name="overrides">Comma-separated capability override entries.</param>
+    public CapabilityOverrideLayer(string overrides)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(overrides);
+        Overrides = overrides;
+    }
+
+    /// <summary>
+    /// Gets capability override entries.
+    /// </summary>
+    public string Overrides { get; }
+
+    /// <summary>
+    /// Applies capability overrides to the specified operator.
+    /// </summary>
+    /// <param name="op">Operator to layer.</param>
+    /// <returns>The layered operator instance.</returns>
+    public Operator Apply(Operator op)
+    {
+        ArgumentNullException.ThrowIfNull(op);
+        ObjectDisposedException.ThrowIf(op.IsInvalid, op);
+
+        var result = NativeMethods.operator_layer_capability_override(op, Overrides);
+        return op.ApplyLayerResult(result);
+    }
+}

bindings/dotnet/OpenDAL/NativeMethods.cs

@@ -154,6 +154,13 @@ internal static partial OpenDALOperatorResult operator_layer_concurrent_limit(
         nuint permits
     );
 
+    [LibraryImport(__DllName, EntryPoint = "operator_layer_capability_override", StringMarshalling = StringMarshalling.Utf8)]
+    [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
+    internal static partial OpenDALOperatorResult operator_layer_capability_override(
+        Operator op,
+        string overrides
+    );
+
     [LibraryImport(__DllName, EntryPoint = "operator_layer_timeout")]
     [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
     internal static partial OpenDALOperatorResult operator_layer_timeout(

bindings/dotnet/src/operator.rs

@@ -410,6 +410,37 @@ fn operator_layer_concurrent_limit_inner(
     Ok(Box::into_raw(Box::new(op.clone().layer(concurrent_limit))) as *mut c_void)
 }
 
+/// Create a new operator layered with capability override behavior.
+///
+/// The current operator is not modified. Returned pointer must be released with
+/// `operator_free`.
+/// # Safety
+///
+/// - `op` must be a valid operator pointer from `operator_construct`.
+/// - `overrides` must be a valid null-terminated UTF-8 string.
+#[unsafe(no_mangle)]
+pub extern "C" fn operator_layer_capability_override(
+    op: *const opendal::Operator,
+    overrides: *const c_char,
+) -> OpendalOperatorResult {
+    match operator_layer_capability_override_inner(op, overrides) {
+        Ok(value) => OpendalOperatorResult::ok(value),
+        Err(error) => OpendalOperatorResult::from_error(error),
+    }
+}
+
+fn operator_layer_capability_override_inner(
+    op: *const opendal::Operator,
+    overrides: *const c_char,
+) -> Result<*mut c_void, OpenDALError> {
+    let op = require_operator(op)?;
+    let overrides = require_cstr(overrides, "capability overrides")?;
+    let layer = opendal::layers::CapabilityOverrideLayer::from_overrides(overrides)
+        .map_err(OpenDALError::from_opendal_error)?;
+
+    Ok(Box::into_raw(Box::new(op.clone().layer(layer))) as *mut c_void)
+}
+
 /// Create a new operator layered with timeout behavior.
 ///
 /// The current operator is not modified. Returned pointer must be released with

bindings/java/src/layer.rs

@@ -17,12 +17,16 @@
 
 use std::time::Duration;
 
+use crate::Result;
+use crate::convert::jstring_to_string;
 use jni::JNIEnv;
 use jni::objects::JClass;
+use jni::objects::JString;
 use jni::sys::jboolean;
 use jni::sys::jfloat;
 use jni::sys::jlong;
 use opendal::Operator;
+use opendal::layers::CapabilityOverrideLayer;
 use opendal::layers::ConcurrentLimitLayer;
 use opendal::layers::RetryLayer;
 
@@ -51,6 +55,30 @@ pub extern "system" fn Java_org_apache_opendal_layer_RetryLayer_doLayer(
     Box::into_raw(Box::new(op.clone().layer(retry))) as jlong
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_apache_opendal_layer_CapabilityOverrideLayer_doLayer(
+    mut env: JNIEnv,
+    _: JClass,
+    op: *mut Operator,
+    overrides: JString,
+) -> jlong {
+    intern_capability_override_layer(&mut env, op, overrides).unwrap_or_else(|e| {
+        e.throw(&mut env);
+        0
+    })
+}
+
+fn intern_capability_override_layer(
+    env: &mut JNIEnv,
+    op: *mut Operator,
+    overrides: JString,
+) -> Result<jlong> {
+    let op = unsafe { &*op };
+    let overrides = jstring_to_string(env, &overrides)?;
+    let layer = CapabilityOverrideLayer::from_overrides(&overrides)?;
+    Ok(Box::into_raw(Box::new(op.clone().layer(layer))) as jlong)
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_apache_opendal_layer_ConcurrentLimitLayer_doLayer(
     _: JNIEnv,

bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java

@@ -2917,16 +2917,19 @@ class S3 implements ServiceConfig {
          */
         public final Boolean disableStatWithOverride;
         /**
-         * <p>Disable write with if match so that opendal will not send write request with if match headers.</p>
-         * <p>For example, Ceph RADOS S3 doesn't support write with if matched.</p>
+         * <p>Deprecated: S3 write with If-Match capability is enabled by default.</p>
+         *
+         * @deprecated S3 write with If-Match capability is enabled by default and this option is no longer needed.
          */
         public final Boolean disableWriteWithIfMatch;
         /**
          * <p>Indicates whether the client agrees to pay for the requests made to the S3 bucket.</p>
          */
         public final Boolean enableRequestPayer;
         /**
-         * <p>is bucket versioning enabled for this bucket</p>
+         * <p>Deprecated: S3 versioning capability is enabled by default.</p>
+         *
+         * @deprecated S3 versioning capability is enabled by default and this option is no longer needed.
          */
         public final Boolean enableVersioning;
         /**
@@ -2939,7 +2942,9 @@ class S3 implements ServiceConfig {
          */
         public final Boolean enableVirtualHostStyle;
         /**
-         * <p>Enable write with append so that opendal will send write request with append headers.</p>
+         * <p>Deprecated: S3 append capability is enabled by default.</p>
+         *
+         * @deprecated S3 append capability is enabled by default and this option is no longer needed.
          */
         public final Boolean enableWriteWithAppend;
         /**

bindings/java/src/main/java/org/apache/opendal/layer/CapabilityOverrideLayer.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.opendal.layer;
+
+import org.apache.opendal.Layer;
+
+/**
+ * Layer that overrides the full capability exposed by an operator.
+ */
+public class CapabilityOverrideLayer extends Layer {
+
+    private final String overrides;
+
+    public CapabilityOverrideLayer(String overrides) {
+        this.overrides = overrides;
+    }
+
+    @Override
+    protected long layer(long nativeOp) {
+        return doLayer(nativeOp, overrides);
+    }
+
+    private static native long doLayer(long nativeHandle, String overrides);
+}

bindings/java/src/test/java/org/apache/opendal/test/behavior/BehaviorExtension.java

@@ -30,6 +30,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.opendal.AsyncOperator;
 import org.apache.opendal.Operator;
+import org.apache.opendal.layer.CapabilityOverrideLayer;
 import org.apache.opendal.layer.RetryLayer;
 import org.junit.jupiter.api.extension.AfterAllCallback;
 import org.junit.jupiter.api.extension.BeforeAllCallback;
@@ -67,6 +68,10 @@ public void beforeAll(ExtensionContext context) {
             @Cleanup
             final AsyncOperator op = AsyncOperator.of(scheme.toLowerCase().replace('_', '-'), config);
             this.asyncOperator = op.layer(RetryLayer.builder().build());
+            final String capabilityOverrides = dotenv.get("OPENDAL_TEST_CAPABILITY_OVERRIDES");
+            if (capabilityOverrides != null && !capabilityOverrides.trim().isEmpty()) {
+                this.asyncOperator = this.asyncOperator.layer(new CapabilityOverrideLayer(capabilityOverrides));
+            }
             this.operator = this.asyncOperator.blocking();
 
             this.scheme = scheme;

bindings/nodejs/generated.d.ts

@@ -207,6 +207,17 @@ export declare class Capability {
   get shared(): boolean
 }
 
+/**
+ * Capability override layer
+ *
+ * Override the full capability exposed by an operator.
+ */
+export declare class CapabilityOverrideLayer {
+  /** Create a new CapabilityOverrideLayer from capability override entries. */
+  constructor(overrides: string)
+  build(): ExternalObject<Layer>
+}
+
 /**
  * Concurrent limit layer
  *

bindings/nodejs/generated.js

@@ -531,6 +531,7 @@ module.exports.BlockingLister = nativeBinding.BlockingLister
 module.exports.BlockingReader = nativeBinding.BlockingReader
 module.exports.BlockingWriter = nativeBinding.BlockingWriter
 module.exports.Capability = nativeBinding.Capability
+module.exports.CapabilityOverrideLayer = nativeBinding.CapabilityOverrideLayer
 module.exports.ConcurrentLimitLayer = nativeBinding.ConcurrentLimitLayer
 module.exports.Entry = nativeBinding.Entry
 module.exports.Layer = nativeBinding.Layer