ccp: run ccp as a thread within mtcp

README.md

@@ -21,7 +21,7 @@ Compling PSIO/DPDK/NETMAP/ONVM driver requires kernel headers.
 - For Debian/Ubuntu, try ``apt-get install linux-headers-$(uname -r)``
 
 Using [CCP](https://ccp-project.github.io/) for congestion control (enabled by
-default), requires building and running a CCP algorithm. If you would like to
+default), requires the CCP library. If you would like to
 disable CCP (ie. use the internal implementation of Reno), simply comment out
 the `#define USE_CCP` in `mtcp/src/include/mtcp.h`.
 
@@ -32,29 +32,20 @@ the `#define USE_CCP` in `mtcp/src/include/mtcp.h`.
     curl https://sh.rustup.rs -sSf | sh -- -y -v --default-toolchain nightly
     ````
 
-2. Build a CCP algorithm. The [generic-cong-avoid](https://github.com/ccp-project/generic-cong-avoid)
-   package implements standard TCP Reno and Cubic, so this is probably best to
-   start with. The same steps can be followed to build any of the other
-   algorithms hosted in the [ccp-project](https://github.com/ccp-project) organization, such as
-   [bbr](https://github.com/ccp-project/bbr).
-
-   ```bash
-   git clone https://github.com/ccp-project/generic-cong-avoid.git
-   cd generic-cong-avoid
-   cargo +nightly build
-   ````
-
-3. Later, after you've built mTCP and started an mTCP application (such as
-   epserver or perf), you must start the CCP binary you just built. If you
-   try to start the CCP process *before* running an mTCP application, it will
-   report a "connection refused" error.
-   
+2. Install the CCP command line utility:
 
     ```bash
-    cd generic-cong-avoid
-    sudo ./target/debug/reno --ipc unix
+    cargo install portus --bin ccp
     ```
 
+3. Build the library (comes with Reno and Cubic by default, use `ccp get` to add others):
+
+    ```
+    ccp makelib
+    ```
+
+4. You will also need to link your application against `-lccp` and `-lstartccp` as demonstrated in apps/example/Makefie.in
+
 
 We have modified the dpdk package to export net_device stat data 
 (for Intel-based Ethernet adapters only) to the OS. To achieve this, we have

apps/example/Makefile.in

@@ -44,7 +44,7 @@ endif
 ifeq ($(DPDK),1)
 DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
 DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
-LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS}
+LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS} -lstartccp
 endif
 
 # onvm-specific variables

apps/perf/Makefile.in

@@ -44,7 +44,7 @@ endif
 ifeq ($(DPDK),1)
 DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt
 DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS})
-LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS}
+LIBS += -g -O3 -pthread -lrt -march=native ${MTCP_FLD}/lib/libmtcp.a -lnuma -lmtcp -lpthread -lrt -ldl -lgmp -L${RTE_SDK}/${RTE_TARGET}/lib ${DPDK_MACHINE_LDFLAGS} -lstartccp
 endif
 
 # onvm-specific variables

configure.ac

@@ -144,6 +144,10 @@ then
 	AC_MSG_ERROR([Packet I/O library is missing. Please set either dpdk or psio or netmap as your I/O lib.])
 fi
 
+AC_CHECK_LIB(startccp, libstartccp_run_forever, [echo "Found CCP library"], [
+    AC_MSG_ERROR(CCP library is missing. Please see X for instructions.)
+])
+
 AM_INIT_AUTOMAKE(mtcp, 3)
 AC_CONFIG_FILES([Makefile])
 AC_OUTPUT(mtcp/src/Makefile)

mtcp/src/config.c

@@ -647,7 +647,17 @@ ParseConfiguration(char *line)
 #endif
 	} else if (strcmp(p, "multiprocess") == 0) {
 		SetMultiProcessSupport(line + strlen(p) + 1);
-	} else {
+    } else if (strcmp(p, "cc") == 0) {
+#if USE_CCP
+        // ignore the parsing done by the second strtok_r so that we can get the full param string
+        *strchr(q, '\0') = ' ';
+        strcpy(CONFIG.cc, q);
+#else
+        TRACE_CONFIG("[WARNING] 'cc' option provided, but CCP not enabled. define USE_CCP!\n");
+        exit(EXIT_FAILURE);
+#endif
+
+    } else {
 		TRACE_CONFIG("Unknown option type: %s\n", line);
 		return -1;
 	}

mtcp/src/core.c

@@ -74,7 +74,8 @@ static pthread_t g_thread[MAX_CPUS] = {0};
 static pthread_t log_thread[MAX_CPUS]  = {0};
 #endif
 #if USE_CCP
-static pthread_t ccp_thread[MAX_CPUS] = {0};
+static pthread_t ccp_run_thread = 0;
+static pthread_t ccp_recv_thread[MAX_CPUS] = {0};
 #endif
 /*----------------------------------------------------------------------------*/
 static sem_t g_init_sem[MAX_CPUS];
@@ -1091,6 +1092,33 @@ InitializeMTCPManager(struct mtcp_thread_context* ctx)
 }
 /*----------------------------------------------------------------------------*/
 #if USE_CCP
+
+uint32_t libstartccp_run_forever(const char *alg_to_run, uint32_t log_fd);
+
+static void *
+CCPRunThread(void *arg) {
+    // Add ipc argument (always unix, so no need for user to provide manually)
+    char args[1024] = {0};
+    int arglen = strlen(CONFIG.cc)-1;
+    strncpy(args, CONFIG.cc, arglen);
+    strncpy(args+arglen, " --ipc=unix", 11);
+    args[arglen+11] = '\0';
+
+    // Open fd for log file
+	FILE *ccp_log = fopen("cc.log", "w");
+	if (ccp_log == NULL) {
+		perror("fopen cc.log");
+		return 0;
+	}
+    TRACE_CCP("starting ccp thread with args: %s\n", args);
+    TRACE_CCP("printing output to ./cc.log\n");
+    libstartccp_run_forever(args, fileno(ccp_log));
+
+    fclose(ccp_log);
+
+    return 0;
+}
+
 static void *
 CCPRecvLoopThread(void *arg) {
 	mtcp_manager_t mtcp = (mtcp_manager_t)arg;
@@ -1190,12 +1218,14 @@ MTCPRunThread(void *arg)
 #if USE_CCP
 	setup_ccp_connection(mtcp);
 
-	if (pthread_create(&ccp_thread[cpu], NULL,
-		CCPRecvLoopThread, (void *)mtcp) != 0)
-	{
-		TRACE_ERROR("Failed to create thread for CCP receive loop on cpu %d\n", cpu);
-		return NULL;
-	}
+    if (cpu == 0 && pthread_create(&ccp_run_thread, NULL, CCPRunThread, (void *)mtcp) != 0) {
+        TRACE_ERROR("Failed to create thread running CCP on cpu 0");
+    }
+    if (pthread_create(&ccp_recv_thread[cpu], NULL, CCPRecvLoopThread, (void *)mtcp) != 0)
+    {
+        TRACE_ERROR("Failed to create thread for CCP receive loop on cpu %d\n", cpu);
+        return NULL;
+    }
 #endif
 
 	// attach (nic device, queue)
@@ -1397,7 +1427,6 @@ mtcp_free_context(mctx_t mctx)
 
 #if USE_CCP
 	destroy_ccp_connection(mtcp);
-	// pthread_join(ccp_thread[ctx->cpu], NULL);
 	close(mtcp->from_ccp);
 	close(mtcp->to_ccp);
 	TRACE_CCP("CCP thread %d joined.\n", mctx->cpu);
@@ -1531,6 +1560,7 @@ mtcp_setconf(const struct mtcp_conf *conf)
 
 	return 0;
 }
+
 /*----------------------------------------------------------------------------*/
 int 
 mtcp_init(const char *config_file)
@@ -1538,6 +1568,8 @@ mtcp_init(const char *config_file)
 	int i;
 	int ret;
 
+    fprintf(stderr, "hello from mtcp_init\n");
+
 	/* getting cpu and NIC */
 	/* set to max cpus only if user has not arbitrarily set it to lower # */
 	num_cpus = (CONFIG.num_cores == 0) ? GetNumCPUs() : CONFIG.num_cores;

mtcp/src/include/mtcp.h

@@ -189,6 +189,9 @@ struct mtcp_config
   	uint16_t onvm_inst;
   	uint16_t onvm_dest;
 #endif
+#if USE_CCP
+    char     cc[1024];
+#endif
 };
 /*----------------------------------------------------------------------------*/
 struct mtcp_context