ccp: run ccp as a thread within mtcp
parent
ac4716540c
commit
a5c826426f
31
README.md
31
README.md
|
@ -21,7 +21,7 @@ Compling PSIO/DPDK/NETMAP/ONVM driver requires kernel headers.
|
||||||
- For Debian/Ubuntu, try ``apt-get install linux-headers-$(uname -r)``
|
- For Debian/Ubuntu, try ``apt-get install linux-headers-$(uname -r)``
|
||||||
|
|
||||||
Using [CCP](https://ccp-project.github.io/) for congestion control (enabled by
|
Using [CCP](https://ccp-project.github.io/) for congestion control (enabled by
|
||||||
default), requires building and running a CCP algorithm. If you would like to
|
default), requires the CCP library. If you would like to
|
||||||
disable CCP (ie. use the internal implementation of Reno), simply comment out
|
disable CCP (ie. use the internal implementation of Reno), simply comment out
|
||||||
the `#define USE_CCP` in `mtcp/src/include/mtcp.h`.
|
the `#define USE_CCP` in `mtcp/src/include/mtcp.h`.
|
||||||
|
|
||||||
|
@ -32,29 +32,20 @@ the `#define USE_CCP` in `mtcp/src/include/mtcp.h`.
|
||||||
curl https://sh.rustup.rs -sSf | sh -- -y -v --default-toolchain nightly
|
curl https://sh.rustup.rs -sSf | sh -- -y -v --default-toolchain nightly
|
||||||
````
|
````
|
||||||
|
|
||||||
2. Build a CCP algorithm. The [generic-cong-avoid](https://github.com/ccp-project/generic-cong-avoid)
|
2. Install the CCP command line utility:
|
||||||
package implements standard TCP Reno and Cubic, so this is probably best to
|
|
||||||
start with. The same steps can be followed to build any of the other
|
|
||||||
algorithms hosted in the [ccp-project](https://github.com/ccp-project) organization, such as
|
|
||||||
[bbr](https://github.com/ccp-project/bbr).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git clone https://github.com/ccp-project/generic-cong-avoid.git
|
|
||||||
cd generic-cong-avoid
|
|
||||||
cargo +nightly build
|
|
||||||
````
|
|
||||||
|
|
||||||
3. Later, after you've built mTCP and started an mTCP application (such as
|
|
||||||
epserver or perf), you must start the CCP binary you just built. If you
|
|
||||||
try to start the CCP process *before* running an mTCP application, it will
|
|
||||||
report a "connection refused" error.
|
|
||||||
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cd generic-cong-avoid
|
cargo install portus --bin ccp
|
||||||
sudo ./target/debug/reno --ipc unix
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
3. Build the library (comes with Reno and Cubic by default, use `ccp get` to add others):
|
||||||
|
|
||||||
|
```
|
||||||
|
ccp makelib
|
||||||
|
```
|
||||||
|
|
||||||
|
4. You will also need to link your application against `-lccp` and `-lstartccp` as demonstrated in apps/example/Makefie.in
|
||||||
|
|
||||||
|
|
||||||
We have modified the dpdk package to export net_device stat data
|
We have modified the dpdk package to export net_device stat data
|
||||||
(for Intel-based Ethernet adapters only) to the OS. To achieve this, we have
|
(for Intel-based Ethernet adapters only) to the OS. To achieve this, we have
|
||||||
|
|
|
@ -44,7 +44,7 @@ endif
|
||||||
ifeq ($(DPDK),1)
|
ifeq ($(DPDK),1)
|
||||||
DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
|
DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
|
||||||
DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
|
DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
|
||||||
LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS}
|
LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS} -lstartccp
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# onvm-specific variables
|
# onvm-specific variables
|
||||||
|
|
|
@ -44,7 +44,7 @@ endif
|
||||||
ifeq ($(DPDK),1)
|
ifeq ($(DPDK),1)
|
||||||
DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
|
DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
|
||||||
DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
|
DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
|
||||||
LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS}
|
LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS} -lstartccp
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# onvm-specific variables
|
# onvm-specific variables
|
||||||
|
|
|
@ -144,6 +144,10 @@ then
|
||||||
AC_MSG_ERROR([Packet I/O library is missing. Please set either dpdk or psio or netmap as your I/O lib.])
|
AC_MSG_ERROR([Packet I/O library is missing. Please set either dpdk or psio or netmap as your I/O lib.])
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
AC_CHECK_LIB(startccp, libstartccp_run_forever, [echo "Found CCP library"], [
|
||||||
|
AC_MSG_ERROR(CCP library is missing. Please see X for instructions.)
|
||||||
|
])
|
||||||
|
|
||||||
AM_INIT_AUTOMAKE(mtcp, 3)
|
AM_INIT_AUTOMAKE(mtcp, 3)
|
||||||
AC_CONFIG_FILES([Makefile])
|
AC_CONFIG_FILES([Makefile])
|
||||||
AC_OUTPUT(mtcp/src/Makefile)
|
AC_OUTPUT(mtcp/src/Makefile)
|
||||||
|
|
|
@ -647,7 +647,17 @@ ParseConfiguration(char *line)
|
||||||
#endif
|
#endif
|
||||||
} else if (strcmp(p, "multiprocess") == 0) {
|
} else if (strcmp(p, "multiprocess") == 0) {
|
||||||
SetMultiProcessSupport(line + strlen(p) + 1);
|
SetMultiProcessSupport(line + strlen(p) + 1);
|
||||||
} else {
|
} else if (strcmp(p, "cc") == 0) {
|
||||||
|
#if USE_CCP
|
||||||
|
// ignore the parsing done by the second strtok_r so that we can get the full param string
|
||||||
|
*strchr(q, '\0') = ' ';
|
||||||
|
strcpy(CONFIG.cc, q);
|
||||||
|
#else
|
||||||
|
TRACE_CONFIG("[WARNING] 'cc' option provided, but CCP not enabled. define USE_CCP!\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
} else {
|
||||||
TRACE_CONFIG("Unknown option type: %s\n", line);
|
TRACE_CONFIG("Unknown option type: %s\n", line);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -74,7 +74,8 @@ static pthread_t g_thread[MAX_CPUS] = {0};
|
||||||
static pthread_t log_thread[MAX_CPUS] = {0};
|
static pthread_t log_thread[MAX_CPUS] = {0};
|
||||||
#endif
|
#endif
|
||||||
#if USE_CCP
|
#if USE_CCP
|
||||||
static pthread_t ccp_thread[MAX_CPUS] = {0};
|
static pthread_t ccp_run_thread = 0;
|
||||||
|
static pthread_t ccp_recv_thread[MAX_CPUS] = {0};
|
||||||
#endif
|
#endif
|
||||||
/*----------------------------------------------------------------------------*/
|
/*----------------------------------------------------------------------------*/
|
||||||
static sem_t g_init_sem[MAX_CPUS];
|
static sem_t g_init_sem[MAX_CPUS];
|
||||||
|
@ -1091,6 +1092,33 @@ InitializeMTCPManager(struct mtcp_thread_context* ctx)
|
||||||
}
|
}
|
||||||
/*----------------------------------------------------------------------------*/
|
/*----------------------------------------------------------------------------*/
|
||||||
#if USE_CCP
|
#if USE_CCP
|
||||||
|
|
||||||
|
uint32_t libstartccp_run_forever(const char *alg_to_run, uint32_t log_fd);
|
||||||
|
|
||||||
|
static void *
|
||||||
|
CCPRunThread(void *arg) {
|
||||||
|
// Add ipc argument (always unix, so no need for user to provide manually)
|
||||||
|
char args[1024] = {0};
|
||||||
|
int arglen = strlen(CONFIG.cc)-1;
|
||||||
|
strncpy(args, CONFIG.cc, arglen);
|
||||||
|
strncpy(args+arglen, " --ipc=unix", 11);
|
||||||
|
args[arglen+11] = '\0';
|
||||||
|
|
||||||
|
// Open fd for log file
|
||||||
|
FILE *ccp_log = fopen("cc.log", "w");
|
||||||
|
if (ccp_log == NULL) {
|
||||||
|
perror("fopen cc.log");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
TRACE_CCP("starting ccp thread with args: %s\n", args);
|
||||||
|
TRACE_CCP("printing output to ./cc.log\n");
|
||||||
|
libstartccp_run_forever(args, fileno(ccp_log));
|
||||||
|
|
||||||
|
fclose(ccp_log);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static void *
|
static void *
|
||||||
CCPRecvLoopThread(void *arg) {
|
CCPRecvLoopThread(void *arg) {
|
||||||
mtcp_manager_t mtcp = (mtcp_manager_t)arg;
|
mtcp_manager_t mtcp = (mtcp_manager_t)arg;
|
||||||
|
@ -1190,12 +1218,14 @@ MTCPRunThread(void *arg)
|
||||||
#if USE_CCP
|
#if USE_CCP
|
||||||
setup_ccp_connection(mtcp);
|
setup_ccp_connection(mtcp);
|
||||||
|
|
||||||
if (pthread_create(&ccp_thread[cpu], NULL,
|
if (cpu == 0 && pthread_create(&ccp_run_thread, NULL, CCPRunThread, (void *)mtcp) != 0) {
|
||||||
CCPRecvLoopThread, (void *)mtcp) != 0)
|
TRACE_ERROR("Failed to create thread running CCP on cpu 0");
|
||||||
{
|
}
|
||||||
TRACE_ERROR("Failed to create thread for CCP receive loop on cpu %d\n", cpu);
|
if (pthread_create(&ccp_recv_thread[cpu], NULL, CCPRecvLoopThread, (void *)mtcp) != 0)
|
||||||
return NULL;
|
{
|
||||||
}
|
TRACE_ERROR("Failed to create thread for CCP receive loop on cpu %d\n", cpu);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// attach (nic device, queue)
|
// attach (nic device, queue)
|
||||||
|
@ -1397,7 +1427,6 @@ mtcp_free_context(mctx_t mctx)
|
||||||
|
|
||||||
#if USE_CCP
|
#if USE_CCP
|
||||||
destroy_ccp_connection(mtcp);
|
destroy_ccp_connection(mtcp);
|
||||||
// pthread_join(ccp_thread[ctx->cpu], NULL);
|
|
||||||
close(mtcp->from_ccp);
|
close(mtcp->from_ccp);
|
||||||
close(mtcp->to_ccp);
|
close(mtcp->to_ccp);
|
||||||
TRACE_CCP("CCP thread %d joined.\n", mctx->cpu);
|
TRACE_CCP("CCP thread %d joined.\n", mctx->cpu);
|
||||||
|
@ -1531,6 +1560,7 @@ mtcp_setconf(const struct mtcp_conf *conf)
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*----------------------------------------------------------------------------*/
|
/*----------------------------------------------------------------------------*/
|
||||||
int
|
int
|
||||||
mtcp_init(const char *config_file)
|
mtcp_init(const char *config_file)
|
||||||
|
@ -1538,6 +1568,8 @@ mtcp_init(const char *config_file)
|
||||||
int i;
|
int i;
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
fprintf(stderr, "hello from mtcp_init\n");
|
||||||
|
|
||||||
/* getting cpu and NIC */
|
/* getting cpu and NIC */
|
||||||
/* set to max cpus only if user has not arbitrarily set it to lower # */
|
/* set to max cpus only if user has not arbitrarily set it to lower # */
|
||||||
num_cpus = (CONFIG.num_cores == 0) ? GetNumCPUs() : CONFIG.num_cores;
|
num_cpus = (CONFIG.num_cores == 0) ? GetNumCPUs() : CONFIG.num_cores;
|
||||||
|
|
|
@ -189,6 +189,9 @@ struct mtcp_config
|
||||||
uint16_t onvm_inst;
|
uint16_t onvm_inst;
|
||||||
uint16_t onvm_dest;
|
uint16_t onvm_dest;
|
||||||
#endif
|
#endif
|
||||||
|
#if USE_CCP
|
||||||
|
char cc[1024];
|
||||||
|
#endif
|
||||||
};
|
};
|
||||||
/*----------------------------------------------------------------------------*/
|
/*----------------------------------------------------------------------------*/
|
||||||
struct mtcp_context
|
struct mtcp_context
|
||||||
|
|
Loading…
Reference in New Issue