Add pkt_tx_delay parameter in config.ini.

It should be default while handling large concurrent requests, otherwise
it will reduce the throughput.
But if there are only a little, you can set it to less than 100, even to
0, to reduce the delay of the connection.

config.ini

@@ -22,6 +22,13 @@ vlan_strip=1
 # unit: microseconds
 idle_sleep=0
 
+# sent packet delay time(0-100) while send less than 32 pkts.
+# default 100 us.
+# if set 0, means send pkts immediately.
+# if set >100, will dealy 100 us.
+# unit: microseconds
+pkt_tx_delay=100
+
 # enabled port list
 #
 # EBNF grammar:

lib/ff_config.c

@@ -491,6 +491,8 @@ ini_parse_handler(void* user, const char* section, const char* name,
         pconfig->dpdk.vlan_strip = atoi(value);
     } else if (MATCH("dpdk", "idle_sleep")) {
         pconfig->dpdk.idle_sleep = atoi(value);
+    } else if (MATCH("dpdk", "pkt_tx_delay")) {
+        pconfig->dpdk.pkt_tx_delay = atoi(value);
     } else if (MATCH("kni", "enable")) {
         pconfig->kni.enable= atoi(value);
     } else if (MATCH("kni", "method")) {
@@ -711,6 +713,7 @@ ff_default_config(struct ff_config *cfg)
     cfg->dpdk.proc_id = -1;
     cfg->dpdk.numa_on = 1;
     cfg->dpdk.promiscuous = 1;
+    cfg->dpdk.pkt_tx_delay = BURST_TX_DRAIN_US;
 
     cfg->freebsd.hz = 100;
     cfg->freebsd.physmem = 1048576*256;

lib/ff_config.h

@@ -39,6 +39,9 @@ extern "C" {
 extern int dpdk_argc;
 extern char *dpdk_argv[DPDK_CONFIG_NUM + 1];
 
+#define MAX_PKT_BURST 32
+#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
+
 struct ff_hw_features {
     uint8_t rx_csum;
     uint8_t rx_lro;
@@ -109,6 +112,9 @@ struct ff_config {
         /* sleep x microseconds when no pkts incomming */
         unsigned idle_sleep;
 
+        /* TX burst queue drain nodelay dalay time */
+        unsigned pkt_tx_delay;
+
         /* list of proc-lcore */
         uint16_t *proc_lcore;
 

lib/ff_dpdk_if.c

@@ -74,6 +74,7 @@ static int kni_accept;
 static int numa_on;
 
 static unsigned idle_sleep;
+static unsigned pkt_tx_delay;
 
 static struct rte_timer freebsd_clock;
 
@@ -760,6 +761,8 @@ ff_dpdk_init(int argc, char **argv)
     numa_on = ff_global_cfg.dpdk.numa_on;
 
     idle_sleep = ff_global_cfg.dpdk.idle_sleep;
+    pkt_tx_delay = ff_global_cfg.dpdk.pkt_tx_delay > BURST_TX_DRAIN_US ? \
+        BURST_TX_DRAIN_US : ff_global_cfg.dpdk.pkt_tx_delay;
 
     init_lcore_conf();
 
@@ -1402,10 +1405,13 @@ main_loop(void *arg)
     int i, j, nb_rx, idle;
     uint16_t port_id, queue_id;
     struct lcore_conf *qconf;
-    const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
-        US_PER_S * BURST_TX_DRAIN_US;
+    uint64_t drain_tsc = 0;
     struct ff_dpdk_if_context *ctx;
 
+    if (pkt_tx_delay) {
+        drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * pkt_tx_delay;
+    }
+
     prev_tsc = 0;
     usch_tsc = 0;
 
@@ -1425,7 +1431,7 @@ main_loop(void *arg)
          * TX burst queue drain
          */
         diff_tsc = cur_tsc - prev_tsc;
-        if (unlikely(diff_tsc > drain_tsc)) {
+        if (unlikely(diff_tsc >= drain_tsc)) {
             for (i = 0; i < qconf->nb_tx_port; i++) {
                 port_id = qconf->tx_port_id[i];
                 if (qconf->tx_mbufs[port_id].len == 0)
@@ -1488,7 +1494,7 @@ main_loop(void *arg)
 
         div_tsc = rte_rdtsc();
 
-        if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc > drain_tsc))) {
+        if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc >= drain_tsc))) {
             usch_tsc = cur_tsc;
             lr->loop(lr->arg);
         }
@@ -1501,7 +1507,7 @@ main_loop(void *arg)
             end_tsc = idle_sleep_tsc;
         }
 
-        end_tsc = rte_rdtsc();
+        // end_tsc = rte_rdtsc();
 
         if (usch_tsc == cur_tsc) {
             usr_tsc = idle_sleep_tsc - div_tsc;

lib/ff_memory.h

@@ -43,9 +43,6 @@ extern "C" {
 #define RX_QUEUE_SIZE 512
 #define TX_QUEUE_SIZE 512
 
-#define MAX_PKT_BURST 32
-#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
-
 /*
  * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send.
  */