diff --git a/apps/lighttpd-1.4.32/INSTALL b/apps/lighttpd-1.4.32/INSTALL index 0ff7912..97816a8 100644 --- a/apps/lighttpd-1.4.32/INSTALL +++ b/apps/lighttpd-1.4.32/INSTALL @@ -58,7 +58,7 @@ Configure the program with the following arguments: $ ./configure --without-bzip2 CFLAGS="-g -O3" \ --with-libmtcp="<$PATH_TO_MTCP_RELEASE_V3>/mtcp/" \ --with-libpsio="<$PATH_TO_MTCP_RELEASE_V3>/io_engine/" \ - --with-libdpdk="<$PATH_TO_MTCP_RELEASE_V3>/$RTE_SDK/$RTE_TARGET" \ + --with-libdpdk="$RTE_SDK/$RTE_TARGET" \ --with-libonvm="<$PATH_TO_ONVM>" \ --enable-netmap ) @@ -75,7 +75,7 @@ The path should be absolute. $ ./configure --without-bzip2 CFLAGS="-g -DINFO -DDBGERR" \ --with-libmtcp="<$PATH_TO_MTCP_RELEASE_V3>/mtcp/" \ --with-libpsio="<$PATH_TO_MTCP_RELEASE_V3>/io_engine/" \ - --with-libdpdk="<$PATH_TO_MTCP_RELEASE_V3>/$RTE_SDK/$RTE_TARGET" \ + --with-libdpdk="$RTE_SDK/$RTE_TARGET" \ --with-libonvm="<$PATH_TO_ONVM>" \ --enable-netmap ) diff --git a/apps/lighttpd-1.4.32/src/mtcp.conf b/apps/lighttpd-1.4.32/src/mtcp.conf index 77e6864..891ccff 100644 --- a/apps/lighttpd-1.4.32/src/mtcp.conf +++ b/apps/lighttpd-1.4.32/src/mtcp.conf @@ -17,7 +17,7 @@ io = dpdk #num_cores = 8 # Core mask -#core_mask = 00000FFF0 +#core_mask = 0000000F0 # Number of memory channels per processor socket (dpdk-only) num_mem_ch = 4 @@ -52,18 +52,23 @@ port = dpdk0 #port = dpdk1 #port = dpdk0 dpdk1 +# Congestion control algorithm +# (only available when configured with --enable-ccp) +# cc = reno +# cc = cubic + # Maximum concurrency per core (default = 10000) -#max_concurrency = 8192 +#max_concurrency = 10000 # Maximum number of socket buffers per core (default = 10000) # Set this to small value if there are many idle connections -#max_num_buffers = 8192 +#max_num_buffers = 10000 # Receive buffer size of sockets; if not set: rcvbuf = sndbuf -#rcvbuf = 8192 +rcvbuf = 8192 # Send buffer size of sockets; if not set: sndbuf = rcvbuf -#sndbuf = 8192 +sndbuf = 8192 # if sndbuf & rcvbuf not set: sndbuf = rcvbuf = 8192 @@ -81,6 +86,6 @@ tcp_timewait = 0 #stat_print = xge1 #------ DPDK ports -------# stat_print = dpdk0 -#stat_print = dpdk0 dpdk1 +#stat_print = dpdk1 ####################################################### diff --git a/mtcp/src/Makefile.in b/mtcp/src/Makefile.in index 642d1fb..f81a0c8 100644 --- a/mtcp/src/Makefile.in +++ b/mtcp/src/Makefile.in @@ -118,11 +118,10 @@ SRCS = core.c tcp_stream.c config.c api.c eventpoll.c socket.c pipe.c \ tcp_util.c eth_in.c ip_in.c tcp_in.c eth_out.c ip_out.c tcp_out.c \ arp.c timer.c cpu.c rss.c addr_pool.c fhash.c memory_mgt.c logger.c debug.c \ tcp_rb_frag_queue.c tcp_ring_buffer.c tcp_send_buffer.c tcp_sb_queue.c tcp_stream_queue.c \ - psio_module.c io_module.c dpdk_module.c netmap_module.c onvm_module.c icmp.c \ - pacing.c clock.c + psio_module.c io_module.c dpdk_module.c netmap_module.c onvm_module.c icmp.c ifeq ($(CCP), 1) -SRCS += ccp.c +SRCS += ccp.c clock.c pacing.c endif OBJS = $(patsubst %.c,%.o,$(SRCS)) diff --git a/mtcp/src/include/mtcp.h b/mtcp/src/include/mtcp.h index c13043b..791acf9 100644 --- a/mtcp/src/include/mtcp.h +++ b/mtcp/src/include/mtcp.h @@ -52,13 +52,11 @@ #define TCP_OPT_TIMESTAMP_ENABLED TRUE // enabled for rtt measure #define TCP_OPT_SACK_ENABLED TRUE // only recv-side implemented -#define RATE_LIMIT_ENABLED FALSE -#define PACING_ENABLED FALSE - /* Only use rate limiting if using CCP */ #if USE_CCP #undef RATE_LIMIT_ENABLED #define RATE_LIMIT_ENABLED TRUE +#define PACING_ENABLED FALSE // The following two logs are for debugging / experiments only, should be turned // off for production use // #define DBGCCP // ccp debug messages diff --git a/mtcp/src/include/tcp_in.h b/mtcp/src/include/tcp_in.h index 31c17ab..2461b5a 100644 --- a/mtcp/src/include/tcp_in.h +++ b/mtcp/src/include/tcp_in.h @@ -70,7 +70,7 @@ #define TCP_MAX_SYN_RETRY 7 #define TCP_MAX_BACKOFF 7 -#define TCP_INIT_CWND 10 +#define TCP_INIT_CWND 2 enum tcp_state { diff --git a/mtcp/src/include/tcp_stream.h b/mtcp/src/include/tcp_stream.h index a8481e1..9dd48d9 100644 --- a/mtcp/src/include/tcp_stream.h +++ b/mtcp/src/include/tcp_stream.h @@ -113,7 +113,9 @@ struct tcp_send_vars /* congestion control variables */ uint32_t cwnd; /* congestion window */ uint32_t ssthresh; /* slow start threshold */ +#if USE_CCP uint32_t missing_seq; +#endif /* timestamp */ uint32_t ts_lastack_sent; /* last ack sent time */ @@ -192,7 +194,9 @@ typedef struct tcp_stream uint32_t snd_nxt; /* send next */ uint32_t rcv_nxt; /* receive next */ - uint32_t seq_at_last_loss; /* the sequence number we left off at before we stopped at wait_for_acks (due to loss) */ +#if USE_CCP + uint32_t seq_at_last_loss; /* the sequence number we left off at before we stopped at wait_for_acks (due to loss) */ +#endif struct tcp_recv_vars *rcvvar; struct tcp_send_vars *sndvar; diff --git a/mtcp/src/tcp_in.c b/mtcp/src/tcp_in.c index 98fc8ba..5d984a7 100644 --- a/mtcp/src/tcp_in.c +++ b/mtcp/src/tcp_in.c @@ -395,22 +395,24 @@ ProcessACK(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts, } } if (!dup) { +#if USE_CCP if (cur_stream->rcvvar->dup_acks >= 3) { TRACE_DBG("passed dup_acks, ack=%u, snd_nxt=%u, last_ack=%u len=%u wl2=%u peer_wnd=%u right=%u\n", ack_seq-sndvar->iss, cur_stream->snd_nxt-sndvar->iss, cur_stream->rcvvar->last_ack_seq-sndvar->iss, payloadlen, cur_stream->rcvvar->snd_wl2-sndvar->iss, sndvar->peer_wnd / sndvar->mss, right_wnd_edge - sndvar->iss); } +#endif cur_stream->rcvvar->dup_acks = 0; cur_stream->rcvvar->last_ack_seq = ack_seq; } - +#if USE_CCP if(cur_stream->wait_for_acks) { TRACE_DBG("got ack, but waiting to send... ack=%u, snd_next=%u cwnd=%u\n", ack_seq-sndvar->iss, cur_stream->snd_nxt-sndvar->iss, sndvar->cwnd / sndvar->mss); } - +#endif /* Fast retransmission */ if (dup && cur_stream->rcvvar->dup_acks == 3) { TRACE_LOSS("Triple duplicated ACKs!! ack_seq: %u\n", ack_seq); @@ -433,7 +435,11 @@ ProcessACK(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts, "ack_seq: %u, snd_una: %u\n", ack_seq, sndvar->snd_una); } +#if USE_CCP sndvar->missing_seq = ack_seq; +#else + cur_stream->snd_nxt = ack_seq; +#endif } /* update congestion control variables */ @@ -472,13 +478,18 @@ ProcessACK(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts, #endif /* TCP_OPT_SACK_ENABLED */ #if RECOVERY_AFTER_LOSS +#if USE_CCP /* updating snd_nxt (when recovered from loss) */ if (TCP_SEQ_GT(ack_seq, cur_stream->snd_nxt) || (cur_stream->wait_for_acks && TCP_SEQ_GT(ack_seq, cur_stream->seq_at_last_loss) #if TCP_OPT_SACK_ENABLED && cur_stream->rcvvar->sacked_pkts == 0 #endif - )) { + )) +#else + if (TCP_SEQ_GT(ack_seq, cur_stream->snd_nxt)) +#endif /* USE_CCP */ + { #if RTM_STAT sndvar->rstat.ack_upd_cnt++; sndvar->rstat.ack_upd_bytes += (ack_seq - cur_stream->snd_nxt); @@ -487,7 +498,9 @@ ProcessACK(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts, cur_stream->sndvar->cwnd = cur_stream->sndvar->ssthresh; TRACE_LOSS("Updating snd_nxt from %u to %u\n", cur_stream->snd_nxt, ack_seq); +#if USE_CCP cur_stream->wait_for_acks = FALSE; +#endif cur_stream->snd_nxt = ack_seq; TRACE_DBG("Sending again..., ack_seq=%u sndlen=%u cwnd=%u\n", ack_seq-sndvar->iss, @@ -499,7 +512,7 @@ ProcessACK(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ts, AddtoSendList(mtcp, cur_stream); } } -#endif +#endif /* RECOVERY_AFTER_LOSS */ rmlen = ack_seq - sndvar->sndbuf->head_seq; uint16_t packets = rmlen / sndvar->eff_mss; diff --git a/mtcp/src/tcp_out.c b/mtcp/src/tcp_out.c index 565715b..7c9352d 100644 --- a/mtcp/src/tcp_out.c +++ b/mtcp/src/tcp_out.c @@ -486,21 +486,25 @@ FlushTCPSendingBuffer(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ } while (1) { +#if USE_CCP if (sndvar->missing_seq) { seq = sndvar->missing_seq; } else { +#endif seq = cur_stream->snd_nxt; +#if USE_CCP } +#endif //seq = cur_stream->snd_nxt; data = sndvar->sndbuf->head + (seq - sndvar->sndbuf->head_seq); len = sndvar->sndbuf->len - (seq - sndvar->sndbuf->head_seq); - +#if USE_CCP // Without this, mm continually drops packets (not sure why, bursting?) -> mtcp sees lots of losses -> throughput dies if(cur_stream->wait_for_acks && TCP_SEQ_GT(cur_stream->snd_nxt, cur_stream->rcvvar->last_ack_seq)) { goto out; } - +#endif /* sanity check */ if (TCP_SEQ_LT(seq, sndvar->sndbuf->head_seq)) { TRACE_ERROR("Stream %d: Invalid sequence to send. " @@ -586,9 +590,11 @@ FlushTCPSendingBuffer(mtcp_manager_t mtcp, tcp_stream *cur_stream, uint32_t cur_ packets = -3; goto out; } +#if USE_CCP if (sndvar->missing_seq) { sndvar->missing_seq = 0; } +#endif packets++; } diff --git a/mtcp/src/tcp_stream.c b/mtcp/src/tcp_stream.c index 281f2cf..6bf89ff 100644 --- a/mtcp/src/tcp_stream.c +++ b/mtcp/src/tcp_stream.c @@ -313,7 +313,9 @@ CreateTCPStream(mtcp_manager_t mtcp, socket_map_t socket, int type, stream->snd_nxt = stream->sndvar->iss; stream->sndvar->snd_una = stream->sndvar->iss; +#if USE_CCP stream->sndvar->missing_seq = 0; +#endif stream->sndvar->snd_wnd = CONFIG.sndbuf_size; stream->rcv_nxt = 0; stream->rcvvar->rcv_wnd = TCP_INITIAL_WINDOW;