Firmware: Add an additional layer of DMA-assisted sample buffering

firmware/common/LPC4320_M4_memory.ld

@@ -26,7 +26,8 @@ MEMORY
 {
 	/* rom is really the shadow region that points to SPI flash or elsewhere */
 	rom (rx)  : ORIGIN = 0x00000000, LENGTH =  96K
-	ram_local1 (rwx) : ORIGIN = 0x10000000, LENGTH =  96K
+	ram_local1 (rwx) : ORIGIN = 0x10000000, LENGTH =  64K
+	ram_usb (rw) : ORIGIN = 0x10010000, LENGTH = 32K
 	ram_local2 (rwx) : ORIGIN = 0x10080000, LENGTH =  32K
 	ram_sleep (rwx) : ORIGIN = 0x10088000, LENGTH = 8K
 }

firmware/common/LPC4330_M4_memory.ld

@@ -26,7 +26,8 @@ MEMORY
 {
 	/* rom is really the shadow region that points to SPI flash or elsewhere */
 	rom (rx)  : ORIGIN = 0x00000000, LENGTH =  128K
-	ram_local1 (rwx) : ORIGIN = 0x10000000, LENGTH =  128K
+	ram_local1 (rwx) : ORIGIN = 0x10000000, LENGTH =  96K
+	ram_usb(rw) : ORIGIN = 0x10018000, LENGTH = 32K
 	ram_local2 (rwx) : ORIGIN = 0x10080000, LENGTH =  64K
 	ram_sleep (rwx) : ORIGIN = 0x10090000, LENGTH = 8K
 }

firmware/common/LPC43xx_M4_memory.ld

@@ -26,13 +26,14 @@ MEMORY
 	rom_flash (rx)  : ORIGIN = 0x80000000, LENGTH =  1M
 	ram_m0 (rwx) : ORIGIN = 0x20000000, LENGTH = 28K
 	ram_shared (rwx) : ORIGIN = 0x20007000, LENGTH =  4K
-	ram_usb (rwx) : ORIGIN = 0x20008000, LENGTH = 32K
-	/* ram_usb: USB buffer. Straddles two blocks of RAM
-	 * to get performance benefit of having two USB buffers addressable
+	ram_samp (rwx) : ORIGIN = 0x20008000, LENGTH = 32K
+	/* ram_samp: Sample buffer. Straddles two blocks of RAM to get
+	 * performance benefit of having two halves addressable
 	 * simultaneously (on two different buses of the AHB multilayer matrix)
 	 */
 }
 
+usb_samp_buffer = ORIGIN(ram_samp);
 usb_bulk_buffer = ORIGIN(ram_usb);
 m0_state = ORIGIN(ram_shared);
 PROVIDE(__ram_m0_start__ = ORIGIN(ram_m0));

firmware/common/usb.h

@@ -29,6 +29,8 @@
 
 #include "usb_type.h"
 
+usb_queue_head_t* usb_queue_head(const uint_fast8_t endpoint_address);
+
 void usb_peripheral_reset(void);
 void usb_phy_enable(void);
 

firmware/hackrf_usb/hackrf_usb.c

@@ -50,7 +50,6 @@
 #include "usb_api_sweep.h"
 #include "usb_api_transceiver.h"
 #include "usb_api_ui.h"
-#include "usb_bulk_buffer.h"
 #include "usb_api_m0_state.h"
 #include "cpld_xc2c.h"
 #include "portapack.h"

firmware/hackrf_usb/usb_api_sweep.c

@@ -25,7 +25,7 @@
 #include <stddef.h>
 #include <hackrf_core.h>
 #include "usb_api_transceiver.h"
-#include "usb_bulk_buffer.h"
+#include "usb_buffer.h"
 #include "usb_api_m0_state.h"
 #include "tuning.h"
 #include "usb_endpoint.h"
@@ -157,7 +157,7 @@ void sweep_mode(uint32_t seq)
 		m0_state.next_mode = M0_MODE_RX;
 
 		// Write metadata to buffer.
-		buffer = &usb_bulk_buffer[phase * 0x4000];
+		buffer = &usb_samp_buffer[phase * 0x4000];
 		*buffer = 0x7f;
 		*(buffer + 1) = 0x7f;
 		*(buffer + 2) = sweep_freq & 0xff;

firmware/hackrf_usb/usb_api_transceiver.c

@@ -27,7 +27,8 @@
 #include "operacake_sctimer.h"
 
 #include <libopencm3/cm3/vector.h>
-#include "usb_bulk_buffer.h"
+#include <libopencm3/lpc43xx/gpdma.h>
+#include "usb_buffer.h"
 #include "usb_api_m0_state.h"
 
 #include "usb_api_cpld.h" // Remove when CPLD update is handled elsewhere
@@ -40,6 +41,7 @@
 #include "usb.h"
 #include "usb_queue.h"
 #include "platform_detect.h"
+#include "gpdma.h"
 
 #include <stddef.h>
 #include <string.h>
@@ -48,6 +50,13 @@
 #include "usb_api_sweep.h"
 
 #define USB_TRANSFER_SIZE 0x4000
+#define DMA_TRANSFER_SIZE 0x2000
+
+#define DMA_CHANNEL 1
+
+#define BUF_HALF_MASK (USB_SAMP_BUFFER_SIZE >> 1)
+
+volatile uint32_t dma_started, dma_pending, usb_started, usb_completed;
 
 typedef struct {
 	uint32_t freq_mhz;
@@ -317,6 +326,11 @@ void transceiver_shutdown(void)
 
 void transceiver_startup(const transceiver_mode_t mode)
 {
+	dma_started = 0;
+	dma_pending = 0;
+	usb_started = 0;
+	usb_completed = 0;
+
 	hackrf_ui()->set_transceiver_mode(mode);
 
 	switch (mode) {
@@ -402,29 +416,83 @@ usb_request_status_t usb_vendor_request_set_rx_overrun_limit(
 	return USB_REQUEST_STATUS_OK;
 }
 
+void transceiver_start_dma(void* src, void* dest, size_t size)
+{
+	uint32_t num_transfers = size >> 2;
+	gpdma_controller_enable();
+	GPDMA_CSRCADDR(DMA_CHANNEL) = (uint32_t) src;
+	GPDMA_CDESTADDR(DMA_CHANNEL) = (uint32_t) dest;
+	GPDMA_CLLI(DMA_CHANNEL) = 0;
+	GPDMA_CCONTROL(DMA_CHANNEL) = GPDMA_CCONTROL_TRANSFERSIZE(num_transfers) |
+		GPDMA_CCONTROL_SBSIZE(7)                        // 256-transfer src bursts
+		| GPDMA_CCONTROL_DBSIZE(7)                      // 256-transfer dst bursts
+		| GPDMA_CCONTROL_SWIDTH(2)                      // 32-bit src transfers
+		| GPDMA_CCONTROL_DWIDTH(2)                      // 32-bit dst transfers
+		| GPDMA_CCONTROL_S(0)                           // AHB Master 0
+		| GPDMA_CCONTROL_D(1)                           // AHB Master 1
+		| GPDMA_CCONTROL_SI(1)                          // increment source
+		| GPDMA_CCONTROL_DI(1)                          // increment destination
+		| GPDMA_CCONTROL_PROT1(0)                       // user mode
+		| GPDMA_CCONTROL_PROT2(0)                       // not bufferable
+		| GPDMA_CCONTROL_PROT3(0)                       // not cacheable
+		| GPDMA_CCONTROL_I(1);                          // interrupt enabled
+	GPDMA_CCONFIG(DMA_CHANNEL) = GPDMA_CCONFIG_FLOWCNTRL(0) // memory-to-memory
+		| GPDMA_CCONFIG_IE(0)                           // no error interrupt
+		| GPDMA_CCONFIG_ITC(1) // terminal count interrupt
+		| GPDMA_CCONFIG_L(0)   // do not lock
+		| GPDMA_CCONFIG_H(0);  // do not halt
+	GPDMA_INTTCCLEAR = (1 << DMA_CHANNEL);
+	nvic_enable_irq(NVIC_DMA_IRQ);
+	gpdma_channel_enable(DMA_CHANNEL);
+	dma_pending = size;
+}
+
+void dma_isr(void)
+{
+	gpdma_channel_disable(DMA_CHANNEL);
+	GPDMA_INTTCCLEAR = (1 << DMA_CHANNEL);
+	m0_state.m4_count += dma_pending;
+	dma_pending = 0;
+}
+
 void transceiver_bulk_transfer_complete(void* user_data, unsigned int bytes_transferred)
 {
 	(void) user_data;
-	m0_state.m4_count += bytes_transferred;
+	usb_completed += bytes_transferred;
 }
 
 void rx_mode(uint32_t seq)
 {
-	uint32_t usb_count = 0;
-
 	transceiver_startup(TRANSCEIVER_MODE_RX);
 
 	baseband_streaming_enable(&sgpio_config);
 
 	while (transceiver_request.seq == seq) {
-		if ((m0_state.m0_count - usb_count) >= USB_TRANSFER_SIZE) {
+		uint32_t data_gathered = m0_state.m0_count;
+		uint32_t dma_completed = m0_state.m4_count;
+		uint32_t data_available = data_gathered - dma_started;
+		uint32_t space_in_use = usb_completed - dma_completed;
+		uint32_t space_available = USB_BULK_BUFFER_SIZE - space_in_use;
+		bool ahb_busy = !((data_gathered ^ dma_started) & BUF_HALF_MASK);
+		if (!dma_pending && !ahb_busy && (data_available >= DMA_TRANSFER_SIZE) &&
+		    (space_available >= DMA_TRANSFER_SIZE)) {
+			uint32_t samp_offset = dma_started & USB_SAMP_BUFFER_MASK;
+			uint32_t bulk_offset = dma_started & USB_BULK_BUFFER_MASK;
+			transceiver_start_dma(
+				&usb_samp_buffer[samp_offset],
+				&usb_bulk_buffer[bulk_offset],
+				DMA_TRANSFER_SIZE);
+			dma_started += DMA_TRANSFER_SIZE;
+		}
+		if ((dma_completed - usb_started) >= USB_TRANSFER_SIZE) {
+			uint32_t bulk_offset = usb_started & USB_BULK_BUFFER_MASK;
 			usb_transfer_schedule_block(
 				&usb_endpoint_bulk_in,
-				&usb_bulk_buffer[usb_count & USB_BULK_BUFFER_MASK],
+				&usb_bulk_buffer[bulk_offset],
 				USB_TRANSFER_SIZE,
 				transceiver_bulk_transfer_complete,
 				NULL);
-			usb_count += USB_TRANSFER_SIZE;
+			usb_started += USB_TRANSFER_SIZE;
 		}
 	}
 
@@ -433,37 +501,131 @@ void rx_mode(uint32_t seq)
 
 void tx_mode(uint32_t seq)
 {
-	unsigned int usb_count = 0;
-	bool started = false;
-
 	transceiver_startup(TRANSCEIVER_MODE_TX);
 
-	// Set up OUT transfer of buffer 0.
-	usb_transfer_schedule_block(
-		&usb_endpoint_bulk_out,
-		&usb_bulk_buffer[0x0000],
-		USB_TRANSFER_SIZE,
-		transceiver_bulk_transfer_complete,
-		NULL);
-	usb_count += USB_TRANSFER_SIZE;
+	// First, make transfers directly into the sample buffer to fill it.
+	for (int i = 0; i < (USB_SAMP_BUFFER_SIZE / USB_TRANSFER_SIZE); i++) {
+		// Set up transfer.
+		usb_transfer_schedule_block(
+			&usb_endpoint_bulk_out,
+			&usb_samp_buffer[usb_started],
+			USB_TRANSFER_SIZE,
+			transceiver_bulk_transfer_complete,
+			NULL);
+		usb_started += USB_TRANSFER_SIZE;
+
+		// Wait for the transfer to complete.
+		while (usb_completed < usb_started) {
+			// Handle the host switching modes before filling the buffer.
+			if (transceiver_request.seq != seq) {
+				transceiver_shutdown();
+				return;
+			}
+		}
+	}
+
+	// Sample buffer is now full. Update DMA counters accordingly.
+	dma_started = USB_SAMP_BUFFER_SIZE;
+	m0_state.m4_count = USB_SAMP_BUFFER_SIZE;
 
+	// Start transmitting samples.
+	baseband_streaming_enable(&sgpio_config);
+
+	// Continue feeding samples to the sample buffer.
 	while (transceiver_request.seq == seq) {
-		if (!started && (m0_state.m4_count == USB_BULK_BUFFER_SIZE)) {
-			// Buffer is now full, start streaming.
-			baseband_streaming_enable(&sgpio_config);
-			started = true;
+		uint32_t data_used = m0_state.m0_count;
+		uint32_t dma_completed = m0_state.m4_count;
+		uint32_t data_available = usb_completed - dma_started;
+		uint32_t space_in_use = dma_started - data_used;
+		uint32_t space_available = USB_SAMP_BUFFER_SIZE - space_in_use;
+		bool ahb_busy = !((data_used ^ dma_started) & BUF_HALF_MASK);
+		if (!dma_pending && !ahb_busy && (data_available >= DMA_TRANSFER_SIZE) &&
+		    (space_available >= DMA_TRANSFER_SIZE)) {
+			uint32_t samp_offset = dma_started & USB_SAMP_BUFFER_MASK;
+			uint32_t bulk_offset = dma_started & USB_BULK_BUFFER_MASK;
+			transceiver_start_dma(
+				&usb_bulk_buffer[bulk_offset],
+				&usb_samp_buffer[samp_offset],
+				DMA_TRANSFER_SIZE);
+			dma_started += DMA_TRANSFER_SIZE;
 		}
-		if ((usb_count - m0_state.m0_count) <= USB_TRANSFER_SIZE) {
+		if ((usb_started - dma_completed) <= USB_TRANSFER_SIZE) {
+			uint32_t bulk_offset = usb_started & USB_BULK_BUFFER_MASK;
 			usb_transfer_schedule_block(
 				&usb_endpoint_bulk_out,
-				&usb_bulk_buffer[usb_count & USB_BULK_BUFFER_MASK],
+				&usb_bulk_buffer[bulk_offset],
 				USB_TRANSFER_SIZE,
 				transceiver_bulk_transfer_complete,
 				NULL);
-			usb_count += USB_TRANSFER_SIZE;
+			usb_started += USB_TRANSFER_SIZE;
 		}
 	}
 
+	// Host has now requested to stop TX. We won't initiate any further USB
+	// transfers into the bulk buffer. However, we should make sure all
+	// data currently in the USB bulk buffer reaches the sample buffer.
+
+	if ((usb_started - usb_completed) > 0) {
+		// We were part way through a 16KB firmware-side transfer when
+		// the transceiver mode change request to stop TX was received.
+		//
+		// We want to include the contents of that partial transfer in
+		// the data we move to the sample buffer.
+		//
+		// The transfer was already stopped by usb_endpoint_flush(),
+		// which was called from request_transceiver_mode().
+		//
+		// We will not have had a callback, and the transfer descriptor
+		// (dTD) will not have been updated, since the transfer did not
+		// complete.
+		//
+		// However, as long as we haven't started a new transfer, we
+		// can retrieve the partial byte count from the transfer
+		// overlay in the endpoint queue head (dQH) (UM10503 25.9.1).
+
+		usb_queue_head_t* const qh =
+			usb_queue_head(usb_endpoint_bulk_out.address);
+		unsigned int bytes_remaining =
+			(qh->total_bytes & USB_TD_DTD_TOKEN_TOTAL_BYTES_MASK) >>
+			USB_TD_DTD_TOKEN_TOTAL_BYTES_SHIFT;
+		unsigned int bytes_transferred = USB_TRANSFER_SIZE - bytes_remaining;
+		usb_completed += bytes_transferred;
+	}
+
+	// Feed the remaining data from the bulk buffer to the sample buffer.
+	// At this point, we also need to handle the case where there is less data
+	// to be transferred to the sample buffer than a full-sized DMA transfer.
+
+	// Any remainder of less than 4 bytes will be ignored; this is the chunk
+	// size of our DMA transfers.
+	while ((usb_completed - m0_state.m4_count) >= 4) {
+		uint32_t data_used = m0_state.m0_count;
+		uint32_t data_available = usb_completed - dma_started;
+		uint32_t space_in_use = dma_started - data_used;
+		uint32_t space_available = USB_SAMP_BUFFER_SIZE - space_in_use;
+		uint32_t samp_offset = dma_started & USB_SAMP_BUFFER_MASK;
+		uint32_t bulk_offset = dma_started & USB_BULK_BUFFER_MASK;
+		bool ahb_busy = !((data_used ^ dma_started) & BUF_HALF_MASK);
+		size_t size = data_available >= DMA_TRANSFER_SIZE ? DMA_TRANSFER_SIZE :
+								    data_available;
+		if (dma_pending || ahb_busy || size > space_available) {
+			continue;
+		}
+		transceiver_start_dma(
+			&usb_bulk_buffer[bulk_offset],
+			&usb_samp_buffer[samp_offset],
+			size);
+		dma_started += size;
+	}
+
+	// Wait for the data in the sample buffer to be transmitted.
+
+	// Any remainder of less than 32 bytes will be ignored; this is
+	// the chunk size used by the M0 core to transfer samples to SGPIO.
+	while ((m0_state.m4_count - m0_state.m0_count) >= 32)
+		;
+
+	// All data received from the host has now been transmitted.
 	transceiver_shutdown();
 }
 

firmware/hackrf_usb/usb_bulk_buffer.h → firmware/hackrf_usb/usb_buffer.h

@@ -21,19 +21,23 @@
  * Boston, MA 02110-1301, USA.
  */
 
-#ifndef __USB_BULK_BUFFER_H__
-#define __USB_BULK_BUFFER_H__
+#ifndef __USB_BUFFER_H__
+#define __USB_BUFFER_H__
 
 #include <stdbool.h>
 #include <stdint.h>
 
+#define USB_SAMP_BUFFER_SIZE 0x8000
+#define USB_SAMP_BUFFER_MASK 0x7FFF
+
 #define USB_BULK_BUFFER_SIZE 0x8000
 #define USB_BULK_BUFFER_MASK 0x7FFF
 
-/* Address of usb_bulk_buffer is set in ldscripts. If you change the name of this
- * variable, it won't be where it needs to be in the processor's address space,
- * unless you also adjust the ldscripts.
+/* Addresses of usb_samp_buffer and usb_bulk_buffer are set in ldscripts. If
+ * you change the name of these variables, they won't be where they need to
+ * be in the processor's address space, unless you also adjust the ldscripts.
  */
+extern uint8_t usb_samp_buffer[USB_SAMP_BUFFER_SIZE];
 extern uint8_t usb_bulk_buffer[USB_BULK_BUFFER_SIZE];
 
-#endif /*__USB_BULK_BUFFER_H__*/
+#endif /*__USB_BUFFER_H__*/