Performance improvements

benches/resamplers.rs

@@ -8,7 +8,7 @@ mod bench_asyncro {
         Async, FixedAsync, PolynomialDegree, Resampler, SincInterpolationType, WindowFunction,
     };
 
-    use rubato::sinc_interpolator::ScalarInterpolator;
+    use rubato::sinc_interpolator::{AnyInterpolator, ScalarInterpolator};
 
     #[cfg(target_arch = "x86_64")]
     use rubato::sinc_interpolator::sinc_interpolator_avx::AvxInterpolator;
@@ -47,7 +47,7 @@ mod bench_asyncro {
                 window,
             );
             let interpolator = unwrap_helper!($($unwrap)* interpolator);
-            let interpolator = Box::new(interpolator);
+            let interpolator: AnyInterpolator<$ft> = interpolator.into();
             let mut resampler = Async::<$ft>::new_with_sinc_interpolator(
                 resample_ratio,
                 1.1,

src/asynchro.rs

@@ -7,7 +7,7 @@ use crate::asynchro_sinc::{
     make_interpolator, InnerSinc, SincInterpolationParameters, SincInterpolationType,
 };
 use crate::error::{ResampleError, ResampleResult, ResamplerConstructionError};
-use crate::sinc_interpolator::SincInterpolator;
+use crate::sinc_interpolator::{AnyInterpolator, SincInterpolator};
 use crate::{get_offsets, get_partial_len, update_mask, Indexing};
 use crate::{validate_buffers, Resampler, Sample};
 
@@ -285,7 +285,7 @@ where
         resample_ratio: f64,
         max_resample_ratio_relative: f64,
         interpolation_type: SincInterpolationType,
-        interpolator: Box<dyn SincInterpolator<T>>,
+        interpolator: AnyInterpolator<T>,
         chunk_size: usize,
         nbr_channels: usize,
         fixed: FixedAsync,

src/asynchro_sinc.rs

@@ -6,7 +6,7 @@ use crate::sinc_interpolator::sinc_interpolator_avx::AvxInterpolator;
 use crate::sinc_interpolator::sinc_interpolator_neon::NeonInterpolator;
 #[cfg(target_arch = "x86_64")]
 use crate::sinc_interpolator::sinc_interpolator_sse::SseInterpolator;
-use crate::sinc_interpolator::{ScalarInterpolator, SincInterpolator};
+use crate::sinc_interpolator::{AnyInterpolator, ScalarInterpolator, SincInterpolator};
 use crate::windows::WindowFunction;
 use crate::Sample;
 use audioadapter::AdapterMut;
@@ -85,7 +85,7 @@ pub fn make_interpolator<T>(
     f_cutoff: f32,
     oversampling_factor: usize,
     window: WindowFunction,
-) -> Box<dyn SincInterpolator<T>>
+) -> AnyInterpolator<T>
 where
     T: Sample,
 {
@@ -100,24 +100,24 @@ where
     if let Ok(interpolator) =
         AvxInterpolator::<T>::new(sinc_len, oversampling_factor, f_cutoff, window)
     {
-        return Box::new(interpolator);
+        return AnyInterpolator::Avx(interpolator);
     }
 
     #[cfg(target_arch = "x86_64")]
     if let Ok(interpolator) =
         SseInterpolator::<T>::new(sinc_len, oversampling_factor, f_cutoff, window)
     {
-        return Box::new(interpolator);
+        return AnyInterpolator::Sse(interpolator);
     }
 
     #[cfg(target_arch = "aarch64")]
     if let Ok(interpolator) =
         NeonInterpolator::<T>::new(sinc_len, oversampling_factor, f_cutoff, window)
     {
-        return Box::new(interpolator);
+        return AnyInterpolator::Neon(interpolator);
     }
 
-    Box::new(ScalarInterpolator::<T>::new(
+    AnyInterpolator::Scalar(ScalarInterpolator::<T>::new(
         sinc_len,
         oversampling_factor,
         f_cutoff,
@@ -161,8 +161,11 @@ where
     yvals[0] + x * (yvals[1] - yvals[0])
 }
 
-pub(crate) struct InnerSinc<T> {
-    pub interpolator: Box<dyn SincInterpolator<T>>,
+pub(crate) struct InnerSinc<T>
+where
+    T: Sample,
+{
+    pub interpolator: AnyInterpolator<T>,
     pub interpolation: SincInterpolationType,
 }
 
@@ -196,6 +199,10 @@ where
                     let frac = idx * oversampling_factor as f64
                         - (idx * oversampling_factor as f64).floor();
                     let frac_offset = t!(frac);
+                    // Warm L1 with the upcoming sinc rows.
+                    for n in &nearest {
+                        self.interpolator.prefetch_sinc(n.1 as usize);
+                    }
                     for (chan, active) in channel_mask.iter().enumerate() {
                         if *active {
                             let buf = &wave_in[chan];
@@ -226,6 +233,9 @@ where
                     let frac = idx * oversampling_factor as f64
                         - (idx * oversampling_factor as f64).floor();
                     let frac_offset = t!(frac);
+                    for n in &nearest {
+                        self.interpolator.prefetch_sinc(n.1 as usize);
+                    }
                     for (chan, active) in channel_mask.iter().enumerate() {
                         if *active {
                             let buf = &wave_in[chan];
@@ -256,6 +266,7 @@ where
                     let frac = idx * oversampling_factor as f64
                         - (idx * oversampling_factor as f64).floor();
                     let frac_offset = t!(frac);
+                    self.interpolator.prefetch_sinc(nearest[1].1 as usize);
                     for (chan, active) in channel_mask.iter().enumerate() {
                         if *active {
                             let buf = &wave_in[chan];

src/sinc_interpolator/mod.rs

@@ -59,6 +59,128 @@ pub(crate) trait SincInterpolator<T>: Send {
 
     /// Get number of sincs used for oversampling.
     fn nbr_sincs(&self) -> usize;
+
+    /// Issue a hardware prefetch hint for the sinc row at `subindex`.
+    /// The default is a no-op; SIMD implementations may override this to bring
+    /// the next sinc row into cache while the current one is being processed.
+    #[inline]
+    fn prefetch_sinc(&self, _subindex: usize) {}
+}
+
+/// A concrete enum over every sinc interpolator implementation.
+///
+/// Replaces a `Box<dyn SincInterpolator<T>>` so that the hot path can dispatch
+/// via a `match` on a known-small set of variants. This enables the compiler to
+/// inline `get_sinc_interpolated`, which in turn unlocks unrolling of the inner
+/// FMA loop and cross-call register reuse.
+#[cfg_attr(feature = "bench_asyncro", visibility::make(pub))]
+pub(crate) enum AnyInterpolator<T>
+where
+    T: Sample,
+{
+    #[cfg(target_arch = "x86_64")]
+    Avx(sinc_interpolator_avx::AvxInterpolator<T>),
+    #[cfg(target_arch = "x86_64")]
+    Sse(sinc_interpolator_sse::SseInterpolator<T>),
+    #[cfg(target_arch = "aarch64")]
+    Neon(sinc_interpolator_neon::NeonInterpolator<T>),
+    Scalar(ScalarInterpolator<T>),
+}
+
+impl<T> SincInterpolator<T> for AnyInterpolator<T>
+where
+    T: Sample,
+{
+    #[inline]
+    fn get_sinc_interpolated(&self, wave: &[T], index: usize, subindex: usize) -> T {
+        match self {
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Avx(i) => i.get_sinc_interpolated(wave, index, subindex),
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Sse(i) => i.get_sinc_interpolated(wave, index, subindex),
+            #[cfg(target_arch = "aarch64")]
+            AnyInterpolator::Neon(i) => i.get_sinc_interpolated(wave, index, subindex),
+            AnyInterpolator::Scalar(i) => i.get_sinc_interpolated(wave, index, subindex),
+        }
+    }
+
+    #[inline]
+    fn nbr_points(&self) -> usize {
+        match self {
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Avx(i) => i.nbr_points(),
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Sse(i) => i.nbr_points(),
+            #[cfg(target_arch = "aarch64")]
+            AnyInterpolator::Neon(i) => i.nbr_points(),
+            AnyInterpolator::Scalar(i) => i.nbr_points(),
+        }
+    }
+
+    #[inline]
+    fn nbr_sincs(&self) -> usize {
+        match self {
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Avx(i) => i.nbr_sincs(),
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Sse(i) => i.nbr_sincs(),
+            #[cfg(target_arch = "aarch64")]
+            AnyInterpolator::Neon(i) => i.nbr_sincs(),
+            AnyInterpolator::Scalar(i) => i.nbr_sincs(),
+        }
+    }
+
+    #[inline]
+    fn prefetch_sinc(&self, subindex: usize) {
+        match self {
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Avx(i) => i.prefetch_sinc(subindex),
+            #[cfg(target_arch = "x86_64")]
+            AnyInterpolator::Sse(i) => i.prefetch_sinc(subindex),
+            #[cfg(target_arch = "aarch64")]
+            AnyInterpolator::Neon(i) => i.prefetch_sinc(subindex),
+            AnyInterpolator::Scalar(i) => i.prefetch_sinc(subindex),
+        }
+    }
+}
+
+impl<T> From<ScalarInterpolator<T>> for AnyInterpolator<T>
+where
+    T: Sample,
+{
+    fn from(value: ScalarInterpolator<T>) -> Self {
+        AnyInterpolator::Scalar(value)
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+impl<T> From<sinc_interpolator_avx::AvxInterpolator<T>> for AnyInterpolator<T>
+where
+    T: Sample,
+{
+    fn from(value: sinc_interpolator_avx::AvxInterpolator<T>) -> Self {
+        AnyInterpolator::Avx(value)
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+impl<T> From<sinc_interpolator_sse::SseInterpolator<T>> for AnyInterpolator<T>
+where
+    T: Sample,
+{
+    fn from(value: sinc_interpolator_sse::SseInterpolator<T>) -> Self {
+        AnyInterpolator::Sse(value)
+    }
+}
+
+#[cfg(target_arch = "aarch64")]
+impl<T> From<sinc_interpolator_neon::NeonInterpolator<T>> for AnyInterpolator<T>
+where
+    T: Sample,
+{
+    fn from(value: sinc_interpolator_neon::NeonInterpolator<T>) -> Self {
+        AnyInterpolator::Neon(value)
+    }
 }
 
 /// A plain scalar interpolator.