issue: 4392235 Fixing corruption on full SQ completion

We tracked the index of last cmpleted WQE, then on each CQE we iterate from the CQE index back until we reach last completed WQE.
However, since this WQE is considered free more than one WQE can be written in its space especially when we fill SQ to maximum.
Consider the following state:
| WQE3 | last_signalled (4 wqebb) | WQE1 | WQE2 |
Now since last_signalled space is considered free we may add more WQE, for instance,
| WQE3 | WQE4 (2wqebb, last_signalled_ptr) | WQE5 (1 wqebb) | WQE6 (1 wqebb) | WQE1 | WQE2 |
Now if we get completion for WQE6, we would complete untill last_signalled (WQE6 and WQE5), leaking the completions for WQE1,2,3,4.
This results in credits and resource leakage.

To resolve this issue, we replace the last_signalled with last_to_complete and keep the size of wqe in wqebbs per each sq-prop.
last_to_complete always points to the left most uncompleted wqe. Given the example above, last_to_complete will point to WQE1.
After completion loop last_to_complete will point to the next WQE index to be completed being it already psted or not.
Given | WQE1 (last_to_complete) | WQE2 | WQE3 |, on CQE for WQE3, we complete WQE3,WQE2,WQE1 and move last_to_complete to WQE4.
We can do this since we know the size of each WQE in wqebbs.

Removing unused reset_inflight_zc_buffers_ctx.

Signed-off-by: Alexander Grissik <[email protected]>

Author: AlexanderGrissik

contrib/jenkins_tests/gtest.sh

@@ -112,6 +112,10 @@ rc=$(($rc+$?))
 eval "${sudo_cmd} $timeout_exe env GTEST_TAP=2 LD_PRELOAD=$gtest_lib $gtest_app $gtest_opt_ipv6 --gtest_filter=ultra_api* --gtest_output=xml:${WORKSPACE}/${prefix}/test-xlio_ultra_api-ipv6.xml"
 rc=$(($rc+$?))
 
+#Special tests
+eval "${sudo_cmd} $timeout_exe env GTEST_TAP=2 LD_PRELOAD=$gtest_lib XLIO_TCP_CC_ALGO=2 XLIO_TCP_NODELAY=1 $gtest_app $gtest_opt --gtest_filter=ultra_api* --gtest_output=xml:${WORKSPACE}/${prefix}/test-xlio_ultra_api_special.xml"
+rc=$(($rc+$?))
+
 # Worker Threads Mode tests
 
 # Worker Threads Mode test filter

src/core/dev/cq_mgr_tx.cpp

@@ -286,9 +286,11 @@ void cq_mgr_tx::handle_sq_wqe_prop(unsigned index)
 
         prev = p;
         p = p->next;
-    } while (p && m_hqtx_ptr->is_sq_wqe_prop_valid(p, prev));
+    } while (prev != m_hqtx_ptr->m_last_sq_wqe_prop_to_complete);
 
     m_p_ring->return_tx_pool_to_global_pool();
     m_hqtx_ptr->credits_return(credits);
-    m_hqtx_ptr->m_sq_wqe_prop_last_signalled = index;
+    m_hqtx_ptr->m_last_sq_wqe_prop_to_complete =
+        &m_hqtx_ptr->m_sq_wqe_idx_to_prop[(index + m_hqtx_ptr->m_sq_wqe_idx_to_prop[index].wqebbs) %
+                                          m_hqtx_ptr->m_tx_num_wr];
 }

src/core/dev/hw_queue_tx.cpp

@@ -440,7 +440,7 @@ void hw_queue_tx::init_queue()
             hwqtx_logerr("Failed allocating m_sq_wqe_idx_to_prop (errno=%d %m)", errno);
             return;
         }
-        m_sq_wqe_prop_last_signalled = m_tx_num_wr - 1;
+        m_last_sq_wqe_prop_to_complete = m_sq_wqe_idx_to_prop;
         m_sq_wqe_prop_last = nullptr;
     }
 
@@ -790,6 +790,7 @@ inline void hw_queue_tx::submit_wqe(mem_buf_desc_t *buf, unsigned credits, uint8
     m_sq_wqe_idx_to_prop[m_sq_wqe_hot_index] = sq_wqe_prop {
         .buf = buf,
         .credits = credits,
+        .wqebbs = wqebbs,
         .ti = ti,
         .next = m_sq_wqe_prop_last,
     };
@@ -1345,26 +1346,6 @@ void hw_queue_tx::trigger_completion_for_all_sent_packets()
     }
 }
 
-void hw_queue_tx::reset_inflight_zc_buffers_ctx(void *ctx)
-{
-    sq_wqe_prop *p = m_sq_wqe_prop_last;
-    sq_wqe_prop *prev;
-    if (p) {
-        unsigned p_i = p - m_sq_wqe_idx_to_prop;
-        if (p_i == m_sq_wqe_prop_last_signalled) {
-            return;
-        }
-        do {
-            mem_buf_desc_t *desc = p->buf;
-            if (desc && desc->tx.zc.ctx == ctx) {
-                desc->tx.zc.ctx = nullptr;
-            }
-            prev = p;
-            p = p->next;
-        } while (p && is_sq_wqe_prop_valid(p, prev));
-    }
-}
-
 uint32_t hw_queue_tx::is_ratelimit_change(struct xlio_rate_limit_t &rate_limit)
 {
     uint32_t rl_changes = 0;

src/core/dev/hw_queue_tx.h

@@ -42,6 +42,8 @@ struct sq_wqe_prop {
     mem_buf_desc_t *buf;
     /* Number of credits (usually number of WQEBBs). */
     unsigned credits;
+    /* Size of the WQE in WQEBBs. */
+    uint8_t wqebbs;
     /* Transport interface (TIS/TIR) current WQE holds reference to. */
     xlio_ti *ti;
     struct sq_wqe_prop *next;
@@ -101,8 +103,6 @@ class hw_queue_tx : public xlio_ti_owner {
     void put_tls_dek(std::unique_ptr<dpcp::tls_dek> &&dek_obj);
 #endif
 
-    void reset_inflight_zc_buffers_ctx(void *ctx);
-
     void credits_return(unsigned credits) { m_sq_free_credits += credits; }
 
     bool credits_get(unsigned credits)
@@ -188,15 +188,6 @@ class hw_queue_tx : public xlio_ti_owner {
         }
     }
 
-    bool is_sq_wqe_prop_valid(sq_wqe_prop *p, sq_wqe_prop *prev)
-    {
-        unsigned p_i = p - m_sq_wqe_idx_to_prop;
-        unsigned prev_i = prev - m_sq_wqe_idx_to_prop;
-        return (p_i != m_sq_wqe_prop_last_signalled) &&
-            ((m_tx_num_wr + p_i - m_sq_wqe_prop_last_signalled) % m_tx_num_wr <
-             (m_tx_num_wr + prev_i - m_sq_wqe_prop_last_signalled) % m_tx_num_wr);
-    }
-
 #if defined(DEFINED_UTLS)
     inline void tls_fill_static_params_wqe(struct mlx5_wqe_tls_static_params_seg *params,
                                            const struct xlio_tls_info *info, uint32_t key_id,
@@ -229,13 +220,24 @@ class hw_queue_tx : public xlio_ti_owner {
     sq_wqe_prop *m_sq_wqe_idx_to_prop = nullptr;
     sq_wqe_prop *m_sq_wqe_prop_last = nullptr;
 
+    /**
+     * @brief Keeps track of the last Send Queue WQE (Work Queue Element) to be completed.
+     *
+     * This pointer is used to mark the last WQE in the chain that needs to be completed
+     * by a completion event (CQE). For example, if WQEs are posted in order | WQE1 | WQE2 | WQE3 |,
+     * m_last_sq_wqe_prop_to_complete will initially point to WQE1. When a CQE is received for WQE3,
+     * the completion logic will process and complete WQE3, WQE2, WQE1 (from right to left) until
+     * reaching m_last_sq_wqe_prop_to_complete. After that, m_last_sq_wqe_prop_to_complete is
+     * updated to point to the index where the next WQE (e.g., WQE4) should be tracked.
+     */
+    sq_wqe_prop *m_last_sq_wqe_prop_to_complete = nullptr;
+
     struct mlx5_eth_wqe (*m_sq_wqes)[] = nullptr;
     struct mlx5_eth_wqe *m_sq_wqe_hot = nullptr;
     uint8_t *m_sq_wqes_end = nullptr;
 
     const uint32_t m_n_sysvar_tx_num_wr_to_signal;
     uint32_t m_tx_num_wr;
-    unsigned m_sq_wqe_prop_last_signalled = 0U;
     unsigned m_sq_free_credits = 0U;
     uint32_t m_n_unsignaled_count = 0U;
     int m_sq_wqe_hot_index = 0;

src/core/dev/ring.h

@@ -191,12 +191,6 @@ class ring {
         NOT_IN_USE(first);
     }
 
-    virtual void reset_inflight_zc_buffers_ctx(ring_user_id_t id, void *ctx)
-    {
-        NOT_IN_USE(id);
-        NOT_IN_USE(ctx);
-    }
-
     // TODO Add id argument for bonding
     virtual bool credits_get(unsigned credits)
     {

src/core/dev/ring_bond.h

@@ -80,10 +80,6 @@ class ring_bond : public ring {
     virtual bool is_tso(void);
     void slave_create(int if_index);
 
-    void reset_inflight_zc_buffers_ctx(ring_user_id_t id, void *ctx)
-    {
-        m_xmit_rings[id]->reset_inflight_zc_buffers_ctx(id, ctx);
-    }
     virtual uint64_t get_rx_cq_out_of_buffer_drop();
 
 protected:

src/core/dev/ring_simple.h

@@ -223,13 +223,6 @@ class ring_simple : public ring_slave {
         m_hqtx->post_dump_wqe(tis, addr, len, lkey, is_first);
     }
 
-    void reset_inflight_zc_buffers_ctx(ring_user_id_t id, void *ctx) override
-    {
-        std::lock_guard<decltype(m_lock_ring_tx)> lock(m_lock_ring_tx);
-        NOT_IN_USE(id);
-        m_hqtx->reset_inflight_zc_buffers_ctx(ctx);
-    }
-
     bool credits_get(unsigned credits) override
     {
         std::lock_guard<decltype(m_lock_ring_tx)> lock(m_lock_ring_tx);

src/core/proto/dst_entry.h

@@ -104,10 +104,6 @@ class dst_entry : public cache_observer, public tostr {
     uint8_t get_tos() const { return m_tos; }
     uint8_t get_ttl_hop_limit() const { return m_ttl_hop_limit; }
     void set_external_vlan_tag(uint16_t vlan_tag) { m_external_vlan_tag = vlan_tag; }
-    void reset_inflight_zc_buffers_ctx(void *ctx)
-    {
-        m_p_ring->reset_inflight_zc_buffers_ctx(m_id, ctx);
-    }
 
     inline bool is_the_same_ifname(const std::string &ifname)
     {

tests/gtest/Makefile.am

@@ -131,7 +131,8 @@ gtest_SOURCES = \
 	xlio_ultra_api/xlio_socket_migrate.cc \
 	xlio_ultra_api/xlio_socket_migrate_2.cc \
 	xlio_ultra_api/xlio_socket_send_receive.cc \
-	xlio_ultra_api/xlio_socket_send_receive_2.cc
+	xlio_ultra_api/xlio_socket_send_receive_2.cc \
+	xlio_ultra_api/xlio_socket_send_receive_full_sq.cc
 
 noinst_HEADERS = \
 	common/tap.h \