Merge branch 'devel' of https://github.com/eunyoung14/mtcp into devel

master
Asim Jamshed 2018-03-17 09:59:56 +09:00
commit cddd83e39d
45 changed files with 1164 additions and 30 deletions

View File

@ -231,6 +231,8 @@ MAKEINFO = @MAKEINFO@
MKDIR_P = @MKDIR_P@
NETMAP = @NETMAP@
OBJEXT = @OBJEXT@
ONVM = @ONVM@
ONVMLIBPATH = @ONVMLIBPATH@
PACKAGE = @PACKAGE@
PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
PACKAGE_NAME = @PACKAGE_NAME@

77
README
View File

@ -16,7 +16,7 @@ We require the following libraries to run mTCP.
- libnuma
- libpthread
- librt
Compling PSIO/DPDK/NETMAP driver requires kernel headers.
Compling PSIO/DPDK/NETMAP/ONVM driver requires kernel headers.
- For Debian/Ubuntu, try apt-get install linux-headers-$(uname -r)
@ -62,7 +62,7 @@ config - sample mTCP configuration files (may not be necessary)
========================================================================
INSTALL GUIDES
========================================================================
mTCP can be prepared in two ways.
mTCP can be prepared in three ways.
- PSIO VERSION -
----------------
@ -179,6 +179,79 @@ mTCP can be prepared in two ways.
6. Run the applications!
- ONVM VERSION -
----------------
Using onvm when running mTCP allows for single node (local) as well as multi-node setups.
A local setup is useful when only 1 machine is available for the experiment.
The onvm configurations are placed in the `.conf` files for mTCP applications.
Example configurations are provided.
**Multicore support**
To use multiple cores launch multiple instances of the application.
If 2 cores (0,1) are available it is possible to run 2 instances of epserver.
To achieve this use the same `onvm_serv` value for both applications.
This way onvm will split the packets between the 2 epservers based on the RSS hash of the packet.
**Before running the applications make sure that onvm_mgr is running.**
No core overlap between applications and onvm_mgr is allowed.
1. Install openNetVM following these instructions
https://github.com/sdnfv/openNetVM/blob/master/docs/Install.md
2. Next bring the dpdk-registered interfaces up. This can be setup using:
# sudo $RTE_SDK/tools/dpdk-setup-iface.sh
3. Create soft links for include/ and lib/ directories inside
empty dpdk/ directory:
# cd dpdk/
# ln -s $RTE_SDK/$RTE_TARGET/lib lib
# ln -s $RTE_SDK/$RTE_TARGET/include include
4. Setup mtcp library
# ./configure --with-dpdk-lib=$<path_to_dpdk> --with-onvm-lib=$<path_to_onvm_lib>
# e.g. ./configure --with-dpdk-lib=`echo $PWD`/dpdk --with-onvm-lib=`echo $ONVM_HOME`/onvm
# make
- By default, mTCP assumes that there are 16 CPUs in your system.
You can set the CPU limit, e.g. on a 32-core system, by using the following command:
# ./configure --with-dpdk-lib=$<path_to_mtcp_release_v3>/dpdk --with-onvm-lib=$<path_to_onvm_lib> CFLAGS="-DMAX_CPUS=32"
Please note that your NIC should support RSS queues equal to the MAX_CPUS value
(since mTCP expects a one-to-one RSS queue to CPU binding).
- In case `./configure' script prints an error, run the
following command; and then re-do step-4 (configure again):
# autoreconf -ivf
- checksum offloading in the NIC is now ENABLED (by default)!!!
- this only works for dpdk at the moment
- use
./configure --with-dpdk-lib=`echo $PWD`/dpdk --disable-hwcsum
to disable checksum offloading.
- check libmtcp.a in mtcp/lib
- check header files in mtcp/include
- check example binary files in apps/example
5. Check the configurations in apps/example
- epserver.conf for server-side configuration
- epwget.conf for client-side configuration
- you may write your own configuration file for your application
6. Run the applications!
- Notes -
When running applications the secondary process may fail to launch
# EAL: FATAL: Cannot init memory
OR
# Cannot mmap memory for rte_config at [0x7ffff7fb6000], got [0x7ffff7e74000] - please use '--base-virtaddr' option
To prevent this use the base virtual address parameter for the ONVM manager.
Example launch options for onvm_mgr with the provided -v argument.
cd openNetVM/onvm
./go.sh 1,2,3 1 -s stdout -v 0x7f000000000
- NETMAP VERSION -
------------------
See README.netmap for details.

View File

@ -16,7 +16,7 @@ We require the following libraries to run mTCP.
- ``libpthread``
- ``librt``
Compling PSIO/DPDK/NETMAP driver requires kernel headers.
Compling PSIO/DPDK/NETMAP/ONVM driver requires kernel headers.
- For Debian/Ubuntu, try ``apt-get install linux-headers-$(uname -r)``
We have modified the dpdk-17.08 package to export net_device stat data
@ -59,7 +59,7 @@ config: sample mTCP configuration files (may not be necessary)
### INSTALL GUIDES ###
mTCP can be prepared in two ways.
mTCP can be prepared in three ways.
***PSIO VERSION***
@ -194,6 +194,79 @@ mTCP can be prepared in two ways.
6. Run the applications!
***ONVM VERSION***
Using onvm when running mTCP allows for single node (local) as well as multi-node setups. A local
setup is useful when only 1 machine is available for the experiment. The onvm configurations are placed
in the `.conf` files for mTCP applications. Example configurations are provided.
**Multicore support**
To use multiple cores launch multiple instances of the application.
If 2 cores (0,1) are available it is possible to run 2 instances of epserver. To achieve this use the same
`onvm_serv` value for both applications. This way onvm will split the packets between the 2 epservers based
on the RSS hash of the packet.
**Before running the applications make sure that onvm_mgr is running.**
*No core overlap between applications and onvm_mgr is allowed*
1. [Install openNetVM following these instructions](https://github.com/sdnfv/openNetVM/blob/master/docs/Install.md)
2. Next bring the dpdk-registered interfaces up. This can be setup using:
```# sudo $RTE_SDK/tools/dpdk-setup-iface.sh```
3. Create soft links for ``include/`` and ``lib/`` directories inside
empty ``dpdk/`` directory:
```bash
# cd dpdk/
# ln -s $RTE_SDK/$RTE_TARGET/lib lib
# ln -s $RTE_SDK/$RTE_TARGET/include include
```
4. Setup mtcp library
```bash
# ./configure --with-dpdk-lib=$<path_to_dpdk> --with-onvm-lib=$<path_to_onvm_lib>
# e.g. ./configure --with-dpdk-lib=`echo $PWD`/dpdk --with-onvm-lib=`echo $ONVM_HOME`/onvm
# make
```
- By default, mTCP assumes that there are 16 CPUs in your system.
You can set the CPU limit, e.g. on a 32-core system, by using the following command:
```bash
# ./configure --with-dpdk-lib=$<path_to_mtcp_release_v3>/dpdk --with-onvm-lib=$<path_to_onvm_lib> CFLAGS="-DMAX_CPUS=32"
```
Please note that your NIC should support RSS queues equal to the MAX_CPUS value
(since mTCP expects a one-to-one RSS queue to CPU binding).
- In case `./configure' script prints an error, run the
following command; and then re-do step-4 (configure again):
```# autoreconf -ivf```
- checksum offloading in the NIC is now ENABLED (by default)!!!
- this only works for dpdk at the moment
- use ```./configure --with-dpdk-lib=`echo $PWD`/dpdk --with-onvm-lib=$<path_to_onvm_lib> --disable-hwcsum``` to disable checksum offloading.
- check libmtcp.a in mtcp/lib
- check header files in mtcp/include
- check example binary files in apps/example
5. Check the configurations in apps/example
- epserver.conf for server-side configuration
- epwget.conf for client-side configuration
- you may write your own configuration file for your application
6. Run the applications!
**Notes**
When running applications the secondary process can fail to launch (```EAL: FATAL: Cannot init memory```, or
``` Cannot mmap memory for rte_config at [0x7ffff7fb6000], got [0x7ffff7e74000] - please use '--base-virtaddr' option```)
To prevent this use the base virtual address parameter for the ONVM manager.
Example launch options for onvm_mgr with the provided -v argument.
```
cd openNetVM/onvm
./go.sh 1,2,3 1 -s stdout -v 0x7f000000000
```
***NETMAP VERSION***
See README.netmap for details.
@ -271,4 +344,4 @@ We tested the DPDK version (polling driver) with Linux-3.13.0 kernel.
Contact: mtcp-user at list.ndsl.kaist.edu
April 2, 2015.
EunYoung Jeong <notav at ndsl.kaist.edu>
M. Asim Jamshed <ajamshed at ndsl.kaist.edu>
M. Asim Jamshed <ajamshed at ndsl.kaist.edu>

View File

@ -5,6 +5,7 @@ CC=@CC@ -g -O3 -Wall -Werror -fgnu89-inline
DPDK=@DPDK@
PS=@PSIO@
NETMAP=@NETMAP@
ONVM=@ONVM@
CFLAGS=@CFLAGS@
# DPDK LIBRARY and HEADER
@ -55,6 +56,16 @@ else
LIBS += -g -O3 -pthread -lrt -march=native -export-dynamic ${MTCP_FLD}/lib/libmtcp.a -L../../dpdk/lib -lnuma -lmtcp -lpthread -lrt -ldl ${DPDK_LIB_FLAGS}
endif
ifeq ($(ONVM),1)
ifeq ($(RTE_TARGET),)
$(error "Please define RTE_TARGET environment variable")
endif
INC += -I@ONVMLIBPATH@/onvm_nflib
INC += -DENABLE_ONVM
LIBS += @ONVMLIBPATH@/onvm_nflib/$(RTE_TARGET)/libonvm.a
endif
ifeq ($V,) # no echo
export MSG=@echo
export HIDE=@

View File

@ -30,7 +30,7 @@ options:
epwget: simple mtcp-epoll-based http request generator
Single-Process, Multi-threaded Usage:
usage: ./epwget URL #requests [-N #cores] [-c concurrency] -f $mtcp_conf
ex) ./epwget 10.0.0.43/example.txt 10000000 -N 8 -c 8000
ex) ./epwget 10.0.0.43/example.txt 10000000 -N 8 -c 8000 -f epwget.conf
Multi-Process, Single-threaded Usage [DPDK-only]
(Master runs on core 0 by default, Slave processes on core 1~N)
@ -57,9 +57,49 @@ notes:
mtcp_setconf() internally to apply the input arguments to the
configuration.
========================================================================
ONVM setups:
The config file provides simple onvm mtcp setups such as:
- simple endpoint server
- Enable `io = onvm`, and `onvm_serv = 1` in epserver.conf
- Run onvm
$ onvm/go.sh 1,2,3 1 -s stdout
- Run epserver
$ sudo ./epserver -p /path/to/www -f epserver.conf -N 1
- local client/server setup
- Enable `io = onvm`, `onvm_serv` = 1, and `onvm_dest = 2` in epserver.conf
- Enable `io = onvm`, `onvm_serv` = 2, and `onvm_dest = 1` in epwget.conf
- Run onvm
$ onvm/go.sh 1,2,3 1 -s stdout
- Run epserver
$ sudo ./epserver -p /path/to/www -f epserver.conf -N 1
- Run epwget
$ sudo ./epwget $SERVER_IP/foo.html 10000000 -N 1 -c 1024 -f epwget.conf
For running multiple servers with different IPs run [UNTESTED]
- onvm nf_router with the route.conf having entries in form of
mtcp_server_IP1 onvm_serv_id1
mtcp_server_IP2 onvm_serv_id2
- epserver with IP1, onvm_serv_id1
- epserver with IP2, onvm_serv_id2
This will function as an endpoint with multiple IPs.
To setup local epwget clients: [UNTESTED]
- set the epserver onvm_dest to onvm nf_router
- set the epwgets onvm_dest to onvm nf_router
- set all the onvm nf_router config epservers/epwget IP->id combinations
To deal with route errors provide a fake config/route.conf, if an
onvm_dest is provided the config route won't be used but will fix errors.
PLEASE NOTE THAT YOU WILL HAVE TO ADD STATIC ARP TABLE ENTRIES TO RUN
MTCP IN ONVM MODE.
========================================================================
Contact: mtcp-user at list.ndsl.kaist.edu
April 2, 2014.
EunYoung Jeong <notav at ndsl.kaist.edu>
M. Asim Jamshed <ajamshed at ndsl.kaist.edu>
M. Asim Jamshed <ajamshed at ndsl.kaist.edu>

View File

@ -1,9 +1,11 @@
############### mtcp configuration file ###############
# The underlying I/O module you want to use. Please
# enable only one out of the two.
# enable only one out of the four.
#io = psio
#io = netmap
#io = onvm
#io = netmap
io = dpdk
# No. of cores setting (enabling this option will override
@ -18,7 +20,25 @@ io = dpdk
# Number of memory channels per processor socket (dpdk-only)
num_mem_ch = 4
# Enable multi-process support (under development)
#--- ONVM specific args ---#
# Service id (required)
#onvm_serv = 1
# Instance id (optional)
#onvm_inst = 1
# Destination id (will forward to another NF)
# If not set will send packets out
#onvm_dest = 2
# Sample ONVM configurations
# Single node epserver <-> epwget
#onvm_serv = 1
#onvm_dest = 2
# Simple endpoint server multi node setup
#onvm_serv = 1
#--------------------------#
# Enable multi-process support
#multiprocess = 1
# Used port (please adjust accordingly)
@ -64,4 +84,4 @@ stat_print = dpdk0
#stat_print = dpdk0:1
#stat_print = dpdk1
#######################################################
#######################################################

View File

@ -1,8 +1,10 @@
############### mtcp configuration file ###############
# The underlying I/O module you want to use. Please
# enable only one out of the two.
# enable only one out of the three.
#io = psio
#io = onvm
#io = netmap
io = dpdk
# No. of cores setting (enabling this option will override
@ -17,6 +19,22 @@ io = dpdk
# Number of memory channels per processor socket (dpdk-only)
num_mem_ch = 4
#--- ONVM specific args ---#
# Service id (required)
#onvm_serv = 2
# Dest id (used to forward traffic to specific NF)
#onvm_dest = 1
# Sample ONVM configurations
# Single node epserver <-> epwget
#onvm_serv = 2
#onvm_dest = 1
# Simple client for multi node setup
#onvm_serv = 1
#--------------------------#
# Used port (please adjust accordingly)
#------ PSIO ports -------#
#port = xge0 xge1
@ -27,7 +45,7 @@ port = dpdk0
#port = dpdk0:0
#port = dpdk0:1
# Enable multi-process support (under development)
# Enable multi-process support
#multiprocess = 1
# Receive buffer size of sockets; if not set: rcvbuf = sndbuf
@ -61,4 +79,4 @@ tcp_timewait = 0
stat_print = dpdk0
#stat_print = dpdk1
#######################################################
#######################################################

View File

@ -51,6 +51,7 @@ Configure the program with the following arguments:
--with-libmtcp=${PATH_TO_LIBMTCP} \
--with-libpsio=${PATH_TO_LIBPSIO} \
--with-libdpdk=${PATH_TO_LIBDPDK} \
--with-libonvm=${PATH_TO_LIBONVM} \
--enable-netmap
(e.g.
@ -58,6 +59,7 @@ Configure the program with the following arguments:
--with-libmtcp="<$PATH_TO_MTCP_RELEASE_V3>/mtcp/" \
--with-libpsio="<$PATH_TO_MTCP_RELEASE_V3>/io_engine/" \
--with-libdpdk="<$PATH_TO_MTCP_RELEASE_V3>/dpdk/" \
--with-libonvm="<$PATH_TO_ONVM>" \
--enable-netmap
)
@ -66,13 +68,15 @@ The path should be absolute.
## For debugging
$ ./configure --without-bzip2 CFLAGS="-g -DINFO -DDBGERR" \
--with-libmtcp=${PATH_TO_LIBMTCP} --with-libpsio=${PATH_TO_LIBPSIO} \
--with-libdpdk=${PATH_TO_LIBDPDK} --enable-netmap
--with-libdpdk=${PATH_TO_LIBDPDK} --with-libonvm=${PATH_TO_LIBONVM} \
--enable-netmap
(e.g.
$ ./configure --without-bzip2 CFLAGS="-g -DINFO -DDBGERR" \
--with-libmtcp="<$PATH_TO_MTCP_RELEASE_V3>/mtcp/" \
--with-libpsio="<$PATH_TO_MTCP_RELEASE_V3>/io_engine/" \
--with-libdpdk="<$PATH_TO_MTCP_RELEASE_V3>/dpdk/" \
--with-libonvm="<$PATH_TO_ONVM>" \
--enable-netmap
)

View File

@ -244,6 +244,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -132,6 +132,9 @@
/* libmtcp support */
#undef HAVE_LIBMTCP
/* libonvm support */
#undef HAVE_LIBONVM
/* libpcre */
#undef HAVE_LIBPCRE

View File

@ -635,6 +635,8 @@ ac_includes_default="\
ac_subst_vars='am__EXEEXT_FALSE
am__EXEEXT_TRUE
LTLIBOBJS
LIBONVM_LIBS
LIBONVM_CFLAGS
LIBDPDK_LIBS
LIBDPDK_CFLAGS
LIBPSIO_LIBS
@ -832,6 +834,7 @@ enable_multithreading
with_libmtcp
with_libpsio
with_libdpdk
with_libonvm
enable_netmap
'
ac_precious_vars='build_alias
@ -1526,6 +1529,7 @@ Optional Packages:
--with-libmtcp[=PATH] Include libmtcp support in PATH/include and PATH/lib
--with-libpsio[=PATH] Include libpsio support in PATH/include and PATH/lib
--with-libdpdk[=PATH] Include libdpdk support in PATH/include and PATH/lib
--with-libonvm[=PATH] Include libonvm support in PATH
Some influential environment variables:
CC C compiler command
@ -16564,6 +16568,37 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for onvm support" >&5
$as_echo_n "checking for onvm support... " >&6; }
# Check whether --with-libonvm was given.
if test "${with_libonvm+set}" = set; then :
withval=$with_libonvm; WITH_LIBONVM=$withval
else
WITH_LIBONVM=no
fi
LIBONVM_CFLAGS=""
LIBONVM_LIBS=""
if test "$WITH_LIBONVM" != "no"; then
LIBONVM_CFLAGS="-I$WITH_LIBONVM/onvm_nflib"
LIBONVM_LIBS="$WITH_LIBONVM/onvm_nflib/$RTE_TARGET/libonvm.a"
CFLAGS="${CFLAGS} ${LIBONVM_CFLAGS}"
$as_echo "#define HAVE_LIBONVM 1" >>confdefs.h
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for netmap support" >&5
$as_echo_n "checking for netmap support... " >&6; }

View File

@ -702,6 +702,27 @@ fi
AC_SUBST(LIBDPDK_CFLAGS)
AC_SUBST(LIBDPDK_LIBS)
AC_MSG_CHECKING(for onvm support)
AC_ARG_WITH(libonvm,
AC_HELP_STRING([--with-libonvm@<:@=PATH@:>@],[Include libonvm support in PATH]),
[WITH_LIBONVM=$withval],[WITH_LIBONVM=no])
LIBONVM_CFLAGS=""
LIBONVM_LIBS=""
if test "$WITH_LIBONVM" != "no"; then
LIBONVM_CFLAGS="-I$WITH_LIBONVM/onvm_nflib"
LIBONVM_LIBS="$WITH_LIBONVM/onvm_nflib/$RTE_TARGET/libonvm.a"
CFLAGS="${CFLAGS} ${LIBONVM_CFLAGS}"
AC_DEFINE([HAVE_LIBONVM], [1], [libonvm support])
AC_MSG_RESULT(yes)
else
AC_MSG_RESULT(no)
fi
AC_SUBST(LIBONVM_CFLAGS)
AC_SUBST(LIBONVM_LIBS)
AC_MSG_CHECKING(for netmap support)
AC_ARG_ENABLE(netmap,

View File

@ -253,6 +253,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -221,6 +221,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -284,7 +284,7 @@ hdr = server.h buffer.h network.h log.h keyvalue.h \
DEFS= @DEFS@ -DHAVE_VERSION_H -DLIBRARY_DIR="\"$(libdir)\"" -DSBIN_DIR="\"$(sbindir)\""
lighttpd_SOURCES = $(src)
lighttpd_LDADD = $(PCRE_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) $(common_libadd) $(SSL_LIB) $(FAM_LIBS) $(LIBEV_LIBS) $(LIBMTCP_LIBS) $(LIBPSIO_LIBS) $(LIBDPDK_LIBS)
lighttpd_LDADD = $(PCRE_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) $(common_libadd) $(SSL_LIB) $(FAM_LIBS) $(LIBEV_LIBS) $(LIBMTCP_LIBS) $(LIBPSIO_LIBS) $(LIBDPDK_LIBS) $(LIBONVM_LIBS)
lighttpd_LDFLAGS = -export-dynamic
lighttpd_CCPFLAGS = $(FAM_CFLAGS) $(LIBEV_CFLAGS) $(LIBMTCP_CFLAGS)

View File

@ -467,7 +467,7 @@ lighttpd_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_2) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1)
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
lighttpd_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
$(lighttpd_LDFLAGS) $(LDFLAGS) -o $@
@ -624,6 +624,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@
@ -894,7 +896,7 @@ hdr = server.h buffer.h network.h log.h keyvalue.h \
version.h
lighttpd_SOURCES = $(src)
lighttpd_LDADD = $(PCRE_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) $(common_libadd) $(SSL_LIB) $(FAM_LIBS) $(LIBEV_LIBS) $(LIBMTCP_LIBS) $(LIBPSIO_LIBS) $(LIBDPDK_LIBS)
lighttpd_LDADD = $(PCRE_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) $(common_libadd) $(SSL_LIB) $(FAM_LIBS) $(LIBEV_LIBS) $(LIBMTCP_LIBS) $(LIBPSIO_LIBS) $(LIBDPDK_LIBS) $(LIBONVM_LIBS)
lighttpd_LDFLAGS = -export-dynamic
lighttpd_CCPFLAGS = $(FAM_CFLAGS) $(LIBEV_CFLAGS) $(LIBMTCP_CFLAGS)
proc_open_SOURCES = proc_open.c buffer.c

View File

@ -1,8 +1,9 @@
############### mtcp configuration file ###############
# The underlying I/O module you want to use. Please
# enable only one out of the two.
# enable only one out of the three.
#io = psio
#io = onvm
io = dpdk
# Enable multi-process support (under development)
@ -12,6 +13,16 @@ io = dpdk
# Number of memory channels per processor socket (dpdk-only)
num_mem_ch = 4
# ONVM specific args
# Service id (required)
#onvm_serv = 1
#
# Instance id (optional)
#onvm_inst = 1
# Destination id (will forward to another NF)
# If not set will send packets out
#onvm_dest = 2
# Used port (please adjust accordingly)
#------ PSIO ports -------#
#port = xge0 xge1

View File

@ -463,6 +463,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -221,6 +221,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -221,6 +221,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -161,6 +161,8 @@ LIBEV_LIBS = @LIBEV_LIBS@
LIBMTCP_CFLAGS = @LIBMTCP_CFLAGS@
LIBMTCP_LIBS = @LIBMTCP_LIBS@
LIBOBJS = @LIBOBJS@
LIBONVM_CFLAGS = @LIBONVM_CFLAGS@
LIBONVM_LIBS = @LIBONVM_LIBS@
LIBPSIO_CFLAGS = @LIBPSIO_CFLAGS@
LIBPSIO_LIBS = @LIBPSIO_LIBS@
LIBS = @LIBS@

View File

@ -4,6 +4,7 @@
# enable only one out of the two.
#io = psio
#io = netmap
#io = onvm
io = dpdk
# No. of cores setting (enabling this option will override
@ -21,6 +22,16 @@ io = dpdk
# Number of memory channels per processor socket (dpdk-only!)
num_mem_ch = 4
#--- ONVM specific args ---#
# Service id (required)
#onvm_serv = 1
# Instance id (optional)
#onvm_inst = 1
# Destination id (will forward to another NF)
# If not set will send packets out
#onvm_dest = 2
#--------------------------#
# Used port (please adjust accordingly)
#------ PSIO ports -------#
#port = xge0 xge1

21
configure vendored
View File

@ -664,10 +664,12 @@ am__isrc
INSTALL_DATA
INSTALL_SCRIPT
INSTALL_PROGRAM
ONVMLIBPATH
PSLIBPATH
DPDKLIBPATH
HWCSUM
NETMAP
ONVM
PSIO
LRO
DPDK
@ -1369,6 +1371,7 @@ Optional Packages:
--without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no)
--with-dpdk-lib path to the dpdk-16.11 install root
--with-psio-lib path to the ioengine install root
--with-onvm-lib path to the onvm install root
Some influential environment variables:
CC C compiler command
@ -4102,6 +4105,9 @@ LRO=0
# Reset PSIO to 0
PSIO=0
# Reset ONVM to 0
ONVM=0
# Reset NETMAP to 0
NETMAP=0
@ -4178,6 +4184,21 @@ if test "x$enable_netmap" = "xyes"; then :
fi
# Check onvm lib path
# Check whether --with-stuff was given.
if test "${with_stuff+set}" = set; then :
withval=$with_stuff;
fi
if test "$with_onvm_lib" != ""
then
ONVMLIBPATH=$with_onvm_lib
ONVM=1
fi
if test "$with_psio_lib" == "" && test "$with_dpdk_lib" == "" && test "x$enable_netmap" = "xno"
then
as_fn_error $? "Packet I/O library is missing. Please set either dpdk or psio or netmap as your I/O lib." "$LINENO" 5

View File

@ -47,6 +47,8 @@ AC_SUBST(DPDK, 0)
AC_SUBST(LRO, 0)
# Reset PSIO to 0
AC_SUBST(PSIO, 0)
# Reset ONVM to 0
AC_SUBST(ONVM, 0)
# Reset NETMAP to 0
AC_SUBST(NETMAP, 0)
# Reset HWCSUM to 1
@ -96,6 +98,14 @@ AS_IF([test "x$enable_netmap" = "xyes"], [
AC_SUBST(NETMAP, 1)
])
# Check onvm lib path
AC_ARG_WITH(stuff, [ --with-onvm-lib path to the onvm install root])
if test "$with_onvm_lib" != ""
then
AC_SUBST(ONVMLIBPATH, $with_onvm_lib)
AC_SUBST(ONVM, 1)
fi
if test "$with_psio_lib" == "" && test "$with_dpdk_lib" == "" && test "x$enable_netmap" = "xno"
then
AC_MSG_ERROR([Packet I/O library is missing. Please set either dpdk or psio or netmap as your I/O lib.])

View File

@ -6,6 +6,7 @@
PS=@PSIO@
DPDK=@DPDK@
NETMAP=@NETMAP@
ONVM=@ONVM@
LRO=@LRO@
HWCSUM=@HWCSUM@
MTCP_LIB_DIR=../lib
@ -13,7 +14,6 @@ MTCP_LIB=libmtcp.a
MTCP_HDR_DIR=../include
MTCP_HDR = mtcp_api.h mtcp_epoll.h
### GCC ###
GCC=@CC@
@ -59,6 +59,16 @@ else
INC += -DDISABLE_NETMAP
endif
ifeq ($(ONVM),1)
ifeq ($(RTE_TARGET),)
$(error "Please define RTE_SDK environment variable")
endif
INC += -I@ONVMLIBPATH@/onvm_nflib
INC += -DENABLE_ONVM
LDFLAGS += @ONVMLIBPATH@/onvm_nflib/$(RTE_TARGET)/libonvm.a
endif
# PacketShader LIBRARY and HEADER
PS_DIR=../../io_engine
INC += -I$(PS_DIR)/include
@ -79,7 +89,7 @@ SRCS = core.c tcp_stream.c config.c api.c eventpoll.c socket.c pipe.c \
tcp_util.c eth_in.c ip_in.c tcp_in.c eth_out.c ip_out.c tcp_out.c \
arp.c timer.c cpu.c rss.c addr_pool.c fhash.c memory_mgt.c logger.c debug.c \
tcp_rb_frag_queue.c tcp_ring_buffer.c tcp_send_buffer.c tcp_sb_queue.c tcp_stream_queue.c \
psio_module.c io_module.c dpdk_module.c netmap_module.c icmp.c
psio_module.c io_module.c dpdk_module.c netmap_module.c onvm_module.c icmp.c
OBJS = $(patsubst %.c,%.o,$(SRCS))
DEPS = $(patsubst %.c,.%.d,$(SRCS))

View File

@ -128,7 +128,8 @@ EnrollRouteTableEntry(char *optstr)
exit(4);
}
#endif
} else if (current_iomodule_func == &dpdk_module_func) {
} else if (current_iomodule_func == &dpdk_module_func ||
current_iomodule_func == &onvm_module_func) {
#ifndef DISABLE_DPDK
for (i = 0; i < num_devices; i++) {
if (strcmp(CONFIG.eths[i].dev_name, dev))
@ -607,6 +608,14 @@ ParseConfiguration(char *line)
AssignIOModule(q);
} else if (strcmp(p, "num_mem_ch") == 0) {
CONFIG.num_mem_ch = mystrtol(q, 10);
#ifdef ENABLE_ONVM
} else if (strcmp(p, "onvm_inst") == 0) {
CONFIG.onvm_inst = mystrtol(q, 10);
} else if (strcmp(p, "onvm_serv") == 0) {
CONFIG.onvm_serv = mystrtol(q, 10);
} else if (strcmp(p, "onvm_dest") == 0) {
CONFIG.onvm_dest = mystrtol(q, 10);
#endif
} else if (strcmp(p, "multiprocess") == 0) {
SetMultiProcessSupport(line + strlen(p) + 1);
} else {
@ -643,7 +652,11 @@ LoadConfiguration(const char *fname)
CONFIG.tcp_timeout = TCP_TIMEOUT;
CONFIG.tcp_timewait = TCP_TIMEWAIT;
CONFIG.num_mem_ch = 0;
#ifdef ENABLE_ONVM
CONFIG.onvm_inst = (uint16_t) -1;
CONFIG.onvm_dest = (uint16_t) -1;
CONFIG.onvm_serv = (uint16_t) -1;
#endif
while (1) {
char *p;
char *temp;

View File

@ -35,6 +35,10 @@
#include <rte_lcore.h>
#endif
#ifdef ENABLE_ONVM
#include "onvm_nflib.h"
#endif
#define PS_CHUNK_SIZE 64
#define RX_THRESH (PS_CHUNK_SIZE * 0.8)
@ -82,6 +86,9 @@ HandleSignal(int signal)
int core;
struct timespec cur_ts;
#ifdef ENABLE_ONVM
onvm_nflib_stop();
#endif
core = sched_getcpu();
clock_gettime(CLOCK_REALTIME, &cur_ts);
@ -913,7 +920,7 @@ InitializeMTCPManager(struct mtcp_thread_context* ctx)
}
mtcp->ctx = ctx;
#ifndef DISABLE_DPDK
#if !defined(DISABLE_DPDK) && !ENABLE_ONVM
char pool_name[RTE_MEMPOOL_NAMESIZE];
sprintf(pool_name, "flow_pool_%d", ctx->cpu);
mtcp->flow_pool = MPCreate(pool_name, sizeof(tcp_stream),

View File

@ -99,6 +99,9 @@ extern io_module_func dpdk_module_func;
/* registered netmap context */
extern io_module_func netmap_module_func;
/* registered onvm context */
extern io_module_func onvm_module_func;
/* Macro to assign IO module */
#define AssignIOModule(m) { \
if (!strcmp(m, "psio")) \
@ -107,6 +110,8 @@ extern io_module_func netmap_module_func;
current_iomodule_func = &dpdk_module_func; \
else if (!strcmp(m, "netmap")) \
current_iomodule_func = &netmap_module_func; \
else if (!strcmp(m, "onvm")) \
current_iomodule_func = &onvm_module_func; \
else \
assert(0); \
}

View File

@ -1,7 +1,7 @@
#ifndef __MEMORY_MGT_H_
#define __MEMORY_MGT_H_
/*----------------------------------------------------------------------------*/
#ifndef DISABLE_DPDK
#if ! defined(DISABLE_DPDK) && !defined(ENABLE_ONVM)
#include <rte_common.h>
#include <rte_mempool.h>
/*----------------------------------------------------------------------------*/

View File

@ -161,6 +161,13 @@ struct mtcp_config
/* adding multi-process support */
uint8_t multi_process;
uint8_t multi_process_is_master;
#ifdef ENABLE_ONVM
/* onvm specific args */
uint16_t onvm_serv;
uint16_t onvm_inst;
uint16_t onvm_dest;
#endif
};
/*----------------------------------------------------------------------------*/
struct mtcp_context

View File

@ -9,12 +9,15 @@
#include "mtcp.h"
#include "fhash.h"
#ifndef TCP_FLAGS
#define TCP_FLAGS
#define TCP_FLAG_FIN 0x01 // 0000 0001
#define TCP_FLAG_SYN 0x02 // 0000 0010
#define TCP_FLAG_RST 0x04 // 0000 0100
#define TCP_FLAG_PSH 0x08 // 0000 1000
#define TCP_FLAG_ACK 0x10 // 0001 0000
#define TCP_FLAG_URG 0x20 // 0010 0000
#endif
#define TCP_FLAG_SACK 0x40 // 0100 0000
#define TCP_FLAG_WACK 0x80 // 1000 0000

View File

@ -27,6 +27,11 @@
/* for getifaddrs */
#include <sys/types.h>
#include <ifaddrs.h>
#ifdef ENABLE_ONVM
/* for onvm */
#include "onvm_nflib.h"
#include "onvm_pkt_helper.h"
#endif
/*----------------------------------------------------------------------------*/
io_module_func *current_iomodule_func = &dpdk_module_func;
#ifndef DISABLE_DPDK
@ -38,6 +43,10 @@ enum rte_proc_type_t eal_proc_type_detect(void);
#define MAX(a, b) ((a)>(b)?(a):(b))
#define MIN(a, b) ((a)<(b)?(a):(b))
/*----------------------------------------------------------------------------*/
/* onvm struct for port info lookup */
extern struct port_info *ports;
#ifndef DISABLE_PSIO
static int
GetNumQueues()
@ -383,6 +392,127 @@ SetInterfaceInfo(char* dev_name_list)
freeifaddrs(ifap);
#endif /* !DISABLE_NETMAP */
} else if (current_iomodule_func == &onvm_module_func) {
#ifdef ENABLE_ONVM
int cpu = CONFIG.num_cores;
uint32_t cpumask = 0;
char cpumaskbuf[10];
char mem_channels[5];
char service[6];
char instance[6];
int ret;
/* get the cpu mask */
for (ret = 0; ret < cpu; ret++)
cpumask = (cpumask | (1 << ret));
sprintf(cpumaskbuf, "%X", cpumask);
/* get the mem channels per socket */
if (CONFIG.num_mem_ch == 0) {
TRACE_ERROR("DPDK module requires # of memory channels "
"per socket parameter!\n");
exit(EXIT_FAILURE);
}
sprintf(mem_channels, "%d", CONFIG.num_mem_ch);
sprintf(service, "%d", CONFIG.onvm_serv);
sprintf(instance, "%d", CONFIG.onvm_inst);
/* initialize the rte env first, what a waste of implementation effort! */
char *argv[] = {"",
"-c",
cpumaskbuf,
"-n",
mem_channels,
"--proc-type=secondary",
"--",
"-r",
service,
instance,
""
};
const int argc = 10;
/*
* re-set getopt extern variable optind.
* this issue was a bitch to debug
* rte_eal_init() internally uses getopt() syscall
* mtcp applications that also use an `external' getopt
* will cause a violent crash if optind is not reset to zero
* prior to calling the func below...
* see man getopt(3) for more details
*/
optind = 0;
/* initialize the dpdk eal env */
ret = onvm_nflib_init(argc, argv, "mtcp_nf");
if (ret < 0)
rte_exit(EXIT_FAILURE, "Invalid EAL args!\n");
/* give me the count of 'detected' ethernet ports */
num_devices = ports->num_ports;
if (num_devices == 0) {
rte_exit(EXIT_FAILURE, "No Ethernet port!\n");
}
struct ifaddrs *ifap;
struct ifaddrs *iter_if;
char *seek;
if (getifaddrs(&ifap) != 0) {
perror("getifaddrs: ");
exit(EXIT_FAILURE);
}
iter_if = ifap;
do {
if (iter_if->ifa_addr->sa_family == AF_INET &&
!set_all_inf &&
(seek=strstr(dev_name_list, iter_if->ifa_name)) != NULL &&
/* check if the interface was not aliased */
*(seek + strlen(iter_if->ifa_name)) != ':') {
struct ifreq ifr;
/* Setting informations */
eidx = CONFIG.eths_num++;
strcpy(CONFIG.eths[eidx].dev_name, iter_if->ifa_name);
strcpy(ifr.ifr_name, iter_if->ifa_name);
/* Create socket */
int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
if (sock == -1) {
perror("socket");
}
/* getting address */
if (ioctl(sock, SIOCGIFADDR, &ifr) == 0 ) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].ip_addr = *(uint32_t *)&sin;
}
for (j = 0; j < ETH_ALEN; j ++) {
CONFIG.eths[eidx].haddr[j] = ports->mac[eidx].addr_bytes[j];
};
/* Net MASK */
if (ioctl(sock, SIOCGIFNETMASK, &ifr) == 0) {
struct in_addr sin = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr;
CONFIG.eths[eidx].netmask = *(uint32_t *)&sin;
}
close(sock);
CONFIG.eths[eidx].ifindex = ports->id[eidx];
devices_attached[num_devices_attached] = CONFIG.eths[eidx].ifindex;
num_devices_attached++;
fprintf(stderr, "Total number of attached devices: %d\n",
num_devices_attached);
fprintf(stderr, "Interface name: %s\n",
iter_if->ifa_name);
}
iter_if = iter_if->ifa_next;
} while (iter_if != NULL);
freeifaddrs(ifap);
#endif /* ENABLE_ONVM */
}
CONFIG.nif_to_eidx = (int*)calloc(MAX_DEVICES, sizeof(int));
@ -416,8 +546,8 @@ FetchEndianType()
#ifndef DISABLE_DPDK
char *argv;
char **argp = &argv;
/* dpdk_module_func logic down below */
dpdk_module_func.dev_ioctl(NULL, CONFIG.eths[0].ifindex, DRV_NAME, (void *)argp);
/* dpdk_module_func/onvm_module_func logic down below */
(*current_iomodule_func).dev_ioctl(NULL, CONFIG.eths[0].ifindex, DRV_NAME, (void *)argp);
if (!strcmp(*argp, "net_i40e"))
return 1;

View File

@ -16,7 +16,7 @@ typedef struct tag_mem_chunk
/*----------------------------------------------------------------------------*/
typedef mem_chunk *mem_chunk_t;
/*----------------------------------------------------------------------------*/
#ifdef DISABLE_DPDK
#if defined(DISABLE_DPDK) || defined(ENABLE_ONVM)
typedef struct mem_pool
{
u_char *mp_startptr; /* start pointer */

572
mtcp/src/onvm_module.c Normal file
View File

@ -0,0 +1,572 @@
/* for io_module_func def'ns */
#include "io_module.h"
#ifdef ENABLE_ONVM
/* for mtcp related def'ns */
#include "mtcp.h"
/* for errno */
#include <errno.h>
/* for logging */
#include "debug.h"
/* for num_devices_* */
#include "config.h"
/* for rte_max_eth_ports */
#include <rte_common.h>
/* for rte_eth_rxconf */
#include <rte_ethdev.h>
/* for delay funcs */
#include <rte_cycles.h>
#include <rte_errno.h>
#define ENABLE_STATS_IOCTL 1
#ifdef ENABLE_STATS_IOCTL
/* for close */
#include <unistd.h>
/* for open */
#include <fcntl.h>
/* for ioctl */
#include <sys/ioctl.h>
#endif /* !ENABLE_STATS_IOCTL */
/* for ip pseudo-chksum */
#include <rte_ip.h>
/* for onvm rings */
#include <onvm_nflib.h>
#include <onvm_pkt_helper.h>
/*----------------------------------------------------------------------------*/
/* Essential macros */
//#define MAX_RX_QUEUE_PER_LCORE MAX_CPUS
//#define MAX_TX_QUEUE_PER_PORT MAX_CPUS
#define PKTMBUF_POOL_NAME "MProc_pktmbuf_pool"
#ifdef ENABLELRO
#define MBUF_SIZE (16384 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
#else
#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
#endif /* !ENABLELRO */
#define NB_MBUF 8192
#define MEMPOOL_CACHE_SIZE 256
//#define RX_IDLE_ENABLE 1
#define RX_IDLE_TIMEOUT 1 /* in micro-seconds */
#define RX_IDLE_THRESH 64
#define MAX_PKT_BURST ((uint16_t)32)/*64*//*128*/
/*
* Configurable number of RX/TX ring descriptors
*/
//#define RTE_TEST_RX_DESC_DEFAULT 128
//#define RTE_TEST_TX_DESC_DEFAULT 128
/*----------------------------------------------------------------------------*/
/* packet memory pool for storing packet bufs */
static struct rte_mempool *pktmbuf_pool = NULL;
//#define DEBUG 1
#ifdef DEBUG
/* ethernet addresses of ports */
static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
#endif
static struct rte_eth_dev_info dev_info[RTE_MAX_ETHPORTS];
struct mbuf_table {
unsigned len; /* length of queued packets */
struct rte_mbuf *m_table[MAX_PKT_BURST];
};
struct dpdk_private_context {
struct mbuf_table rmbufs[RTE_MAX_ETHPORTS];
struct mbuf_table wmbufs[RTE_MAX_ETHPORTS];
struct rte_mempool *pktmbuf_pool;
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
#ifdef RX_IDLE_ENABLE
uint8_t rx_idle;
#endif
#ifdef ENABLELRO
struct rte_mbuf *cur_rx_m;
#endif
#ifdef ENABLE_STATS_IOCTL
int fd;
uint32_t cur_ts;
#endif /* !ENABLE_STATS_IOCTL */
} __rte_cache_aligned;
/* onvm structs */
struct onvm_nf_info *nf_info;
struct rte_ring *rx_ring;
struct rte_ring *tx_ring;
volatile struct onvm_nf *nf;
#ifdef ENABLE_STATS_IOCTL
/**
* stats struct passed on from user space to the driver
*/
struct stats_struct {
uint64_t tx_bytes;
uint64_t tx_pkts;
uint64_t rx_bytes;
uint64_t rx_pkts;
uint64_t rmiss;
uint64_t rerr;
uint64_t terr;
uint8_t qid;
uint8_t dev;
};
#endif /* !ENABLE_STATS_IOCTL */
/*----------------------------------------------------------------------------*/
void
onvm_init_handle(struct mtcp_thread_context *ctxt)
{
struct dpdk_private_context *dpc;
int i, j;
char mempool_name[20];
/* create and initialize private I/O module context */
ctxt->io_private_context = calloc(1, sizeof(struct dpdk_private_context));
if (ctxt->io_private_context == NULL) {
TRACE_ERROR("Failed to initialize ctxt->io_private_context: "
"Can't allocate memory\n");
exit(EXIT_FAILURE);
}
sprintf(mempool_name, "mbuf_pool-%d", ctxt->cpu);
dpc = (struct dpdk_private_context *)ctxt->io_private_context;
dpc->pktmbuf_pool = pktmbuf_pool;
/* Complete onvm handshake */
onvm_nflib_nf_ready(nf_info);
/* Initialize onvm rings*/
nf = onvm_nflib_get_nf(nf_info->instance_id);
rx_ring = nf->rx_q;
tx_ring = nf->tx_q;
/* set wmbufs correctly */
for (j = 0; j < num_devices_attached; j++) {
/* Allocate wmbufs for each registered port */
for (i = 0; i < MAX_PKT_BURST; i++) {
dpc->wmbufs[j].m_table[i] = rte_pktmbuf_alloc(pktmbuf_pool);
if (dpc->wmbufs[j].m_table[i] == NULL) {
TRACE_ERROR("Failed to allocate %d:wmbuf[%d] on device %d!\n",
ctxt->cpu, i, j);
exit(EXIT_FAILURE);
}
}
/* set mbufs queue length to 0 to begin with */
dpc->wmbufs[j].len = 0;
}
#ifdef ENABLE_STATS_IOCTL
dpc->fd = open("/dev/dpdk-iface", O_RDWR);
if (dpc->fd == -1) {
TRACE_ERROR("Can't open /dev/dpdk-iface for context->cpu: %d! "
"Are you using mlx4/mlx5 driver?\n",
ctxt->cpu);
}
#endif /* !ENABLE_STATS_IOCTL */
}
/*----------------------------------------------------------------------------*/
int
onvm_link_devices(struct mtcp_thread_context *ctxt)
{
/* linking takes place during mtcp_init() */
return 0;
}
/*----------------------------------------------------------------------------*/
void
onvm_release_pkt(struct mtcp_thread_context *ctxt, int ifidx, unsigned char *pkt_data, int len)
{
/*
* do nothing over here - memory reclamation
* will take place in onvm_recv_pkts
*/
}
/*----------------------------------------------------------------------------*/
int
onvm_send_pkts(struct mtcp_thread_context *ctxt, int nif)
{
struct dpdk_private_context *dpc;
mtcp_manager_t mtcp;
int ret, i;
struct onvm_pkt_meta* meta;
int ifidx;
ifidx = nif;
dpc = (struct dpdk_private_context *)ctxt->io_private_context;
mtcp = ctxt->mtcp_manager;
ret = 0;
/* if there are packets in the queue... flush them out to the wire */
if (dpc->wmbufs[nif].len >/*= MAX_PKT_BURST*/ 0) {
struct rte_mbuf **pkts;
#ifdef ENABLE_STATS_IOCTL
struct rte_eth_stats stats;
struct stats_struct ss;
#endif /* !ENABLE_STATS_IOCTL */
int cnt = dpc->wmbufs[nif].len;
pkts = dpc->wmbufs[nif].m_table;
#ifdef NETSTAT
mtcp->nstat.tx_packets[nif] += cnt;
#ifdef ENABLE_STATS_IOCTL
/* only pass stats after >= 1 sec interval */
if (abs(mtcp->cur_ts - dpc->cur_ts) >= 1000 &&
likely(dpc->fd >= 0)) {
/* rte_get_stats is global func, use only for 1 core */
if (ctxt->cpu == 0) {
rte_eth_stats_get(CONFIG.eths[ifidx].ifindex, &stats);
ss.rmiss = stats.imissed;
ss.rerr = stats.ierrors;
ss.terr = stats.oerrors;
} else
ss.rmiss = ss.rerr = ss.terr = 0;
ss.tx_pkts = mtcp->nstat.tx_packets[ifidx];
ss.tx_bytes = mtcp->nstat.tx_bytes[ifidx];
ss.rx_pkts = mtcp->nstat.rx_packets[ifidx];
ss.rx_bytes = mtcp->nstat.rx_bytes[ifidx];
ss.qid = ctxt->cpu;
ss.dev = CONFIG.eths[ifidx].ifindex;
/* pass the info now */
ioctl(dpc->fd, 0, &ss);
dpc->cur_ts = mtcp->cur_ts;
if (ctxt->cpu == 0)
rte_eth_stats_reset(CONFIG.eths[ifidx].ifindex);
}
#endif /* !ENABLE_STATS_IOCTL */
#endif
for(i=0;i<cnt;i++){
meta = onvm_get_pkt_meta(pkts[i]);
if (CONFIG.onvm_dest == (uint16_t) -1){
meta->action = ONVM_NF_ACTION_OUT;
meta->destination = CONFIG.eths[nif].ifindex;
}
else {
meta->action = ONVM_NF_ACTION_TONF;
meta->destination = CONFIG.onvm_dest;
}
}
ret = rte_ring_enqueue_bulk(tx_ring, (void * const*)pkts ,cnt, NULL);
if(cnt>0 && ret== 0) {
TRACE_ERROR("Dropped %d packets", cnt);
nf->stats.tx_drop += cnt;
}
nf->stats.tx += cnt;
/* time to allocate fresh mbufs for the queue */
for (i = 0; i < dpc->wmbufs[nif].len; i++) {
dpc->wmbufs[nif].m_table[i] = rte_pktmbuf_alloc(pktmbuf_pool);
/* error checking */
if (unlikely(dpc->wmbufs[nif].m_table[i] == NULL)) {
TRACE_ERROR("Failed to allocate %d:wmbuf[%d] on device %d!\n",
ctxt->cpu, i, nif);
exit(EXIT_FAILURE);
}
}
/* reset the len of mbufs var after flushing of packets */
dpc->wmbufs[nif].len = 0;
}
return ret;
}
/*----------------------------------------------------------------------------*/
uint8_t *
onvm_get_wptr(struct mtcp_thread_context *ctxt, int nif, uint16_t pktsize)
{
struct dpdk_private_context *dpc;
mtcp_manager_t mtcp;
struct rte_mbuf *m;
uint8_t *ptr;
int len_of_mbuf;
dpc = (struct dpdk_private_context *) ctxt->io_private_context;
mtcp = ctxt->mtcp_manager;
/* sanity check */
if (unlikely(dpc->wmbufs[nif].len == MAX_PKT_BURST))
return NULL;
len_of_mbuf = dpc->wmbufs[nif].len;
m = dpc->wmbufs[nif].m_table[len_of_mbuf];
/* retrieve the right write offset */
ptr = (void *)rte_pktmbuf_mtod(m, struct ether_hdr *);
m->pkt_len = m->data_len = pktsize;
m->nb_segs = 1;
m->next = NULL;
#ifdef NETSTAT
mtcp->nstat.tx_bytes[nif] += pktsize + 24;
#endif
/* increment the len_of_mbuf var */
dpc->wmbufs[nif].len = len_of_mbuf + 1;
return (uint8_t *)ptr;
}
/*----------------------------------------------------------------------------*/
static inline void
free_pkts(struct rte_mbuf **mtable, unsigned len)
{
int i;
/* free the freaking packets */
for (i = 0; i < len; i++) {
rte_pktmbuf_free(mtable[i]);
RTE_MBUF_PREFETCH_TO_FREE(mtable[i+1]);
}
}
/*----------------------------------------------------------------------------*/
int32_t
onvm_recv_pkts(struct mtcp_thread_context *ctxt, int ifidx)
{
struct dpdk_private_context *dpc;
int ret;
void *pkts[MAX_PKT_BURST];
int i;
dpc = (struct dpdk_private_context *) ctxt->io_private_context;
if (dpc->rmbufs[ifidx].len != 0) {
free_pkts(dpc->rmbufs[ifidx].m_table, dpc->rmbufs[ifidx].len);
dpc->rmbufs[ifidx].len = 0;
}
ret = rte_ring_dequeue_burst(rx_ring, pkts, MAX_PKT_BURST, NULL);
for(i=0;i<ret;i++) {
dpc->pkts_burst[i]=(struct rte_mbuf*)pkts[i];
}
#ifdef RX_IDLE_ENABLE
dpc->rx_idle = (likely(ret != 0)) ? 0 : dpc->rx_idle + 1;
#endif
dpc->rmbufs[ifidx].len = ret;
return ret;
}
/*----------------------------------------------------------------------------*/
uint8_t *
onvm_get_rptr(struct mtcp_thread_context *ctxt, int ifidx, int index, uint16_t *len)
{
struct dpdk_private_context *dpc;
struct rte_mbuf *m;
uint8_t *pktbuf;
dpc = (struct dpdk_private_context *) ctxt->io_private_context;
m = dpc->pkts_burst[index];
//rte_prefetch0(rte_pktmbuf_mtod(m, void *));
*len = m->pkt_len;
pktbuf = rte_pktmbuf_mtod(m, uint8_t *);
/* enqueue the pkt ptr in mbuf */
dpc->rmbufs[ifidx].m_table[index] = m;
/* verify checksum values from ol_flags */
if ((m->ol_flags & (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD)) != 0) {
TRACE_ERROR("%s(%p, %d, %d): mbuf with invalid checksum: "
"%p(%lu);\n",
__func__, ctxt, ifidx, index, m, m->ol_flags);
pktbuf = NULL;
}
#ifdef ENABLELRO
dpc->cur_rx_m = m;
#endif /* ENABLELRO */
return pktbuf;
}
/*----------------------------------------------------------------------------*/
int32_t
onvm_select(struct mtcp_thread_context *ctxt)
{
#ifdef RX_IDLE_ENABLE
struct dpdk_private_context *dpc;
dpc = (struct dpdk_private_context *) ctxt->io_private_context;
if (dpc->rx_idle > RX_IDLE_THRESH) {
dpc->rx_idle = 0;
usleep(RX_IDLE_TIMEOUT);
}
#endif
return 0;
}
/*----------------------------------------------------------------------------*/
void
onvm_destroy_handle(struct mtcp_thread_context *ctxt)
{
struct dpdk_private_context *dpc;
int i;
dpc = (struct dpdk_private_context *) ctxt->io_private_context;
/* free wmbufs */
for (i = 0; i < num_devices_attached; i++)
free_pkts(dpc->wmbufs[i].m_table, MAX_PKT_BURST);
#ifdef ENABLE_STATS_IOCTL
/* free fd */
if (dpc->fd >= 0)
close(dpc->fd);
#endif /* !ENABLE_STATS_IOCTL */
/* free it all up */
free(dpc);
}
/*----------------------------------------------------------------------------*/
void
onvm_load_module(void)
{
int i, portid;
pktmbuf_pool=rte_mempool_lookup(PKTMBUF_POOL_NAME);
if (pktmbuf_pool == NULL){
rte_exit(EXIT_FAILURE, "Cannot init mbuf pool, errno: %d\n",
rte_errno);
}
for (i = 0; i < num_devices_attached; ++i) {
/* get portid form the index of attached devices */
portid = devices_attached[i];
/* check port capabilities */
rte_eth_dev_info_get(portid, &dev_info[portid]);
}
}
/*----------------------------------------------------------------------------*/
int32_t
onvm_dev_ioctl(struct mtcp_thread_context *ctx, int nif, int cmd, void *argp)
{
struct dpdk_private_context *dpc;
struct rte_mbuf *m;
int len_of_mbuf;
struct iphdr *iph;
struct tcphdr *tcph;
void **argpptr = (void **)argp;
#ifdef ENABLELRO
uint8_t *payload, *to;
int seg_off;
#endif
if (cmd == DRV_NAME) {
*argpptr = (void *)dev_info[nif].driver_name;
return 0;
}
int eidx = CONFIG.nif_to_eidx[nif];
iph = (struct iphdr *)argp;
dpc = (struct dpdk_private_context *)ctx->io_private_context;
len_of_mbuf = dpc->wmbufs[eidx].len;
switch (cmd) {
case PKT_TX_IP_CSUM:
if ((dev_info[nif].tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) == 0)
goto dev_ioctl_err;
m = dpc->wmbufs[eidx].m_table[len_of_mbuf - 1];
m->ol_flags = PKT_TX_IP_CKSUM | PKT_TX_IPV4;
m->l2_len = sizeof(struct ether_hdr);
m->l3_len = (iph->ihl<<2);
break;
case PKT_TX_TCP_CSUM:
if ((dev_info[nif].tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) == 0)
goto dev_ioctl_err;
m = dpc->wmbufs[eidx].m_table[len_of_mbuf - 1];
tcph = (struct tcphdr *)((unsigned char *)iph + (iph->ihl<<2));
m->ol_flags |= PKT_TX_TCP_CKSUM;
tcph->check = rte_ipv4_phdr_cksum((struct ipv4_hdr *)iph, m->ol_flags);
break;
#ifdef ENABLELRO
case PKT_RX_TCP_LROSEG:
m = dpc->cur_rx_m;
//if (m->next != NULL)
// rte_prefetch0(rte_pktmbuf_mtod(m->next, void *));
iph = rte_pktmbuf_mtod_offset(m, struct iphdr *, sizeof(struct ether_hdr));
tcph = (struct tcphdr *)((u_char *)iph + (iph->ihl << 2));
payload = (uint8_t *)tcph + (tcph->doff << 2);
seg_off = m->data_len -
sizeof(struct ether_hdr) - (iph->ihl << 2) -
(tcph->doff << 2);
to = (uint8_t *) argp;
m = m->next;
memcpy(to, payload, seg_off);
while (m != NULL) {
//if (m->next != NULL)
// rte_prefetch0(rte_pktmbuf_mtod(m->next, void *));
memcpy(to + seg_off,
rte_pktmbuf_mtod(m, uint8_t *),
m->data_len);
seg_off += m->data_len;
m = m->next;
}
break;
#endif
case PKT_TX_TCPIP_CSUM:
if ((dev_info[nif].tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) == 0)
goto dev_ioctl_err;
if ((dev_info[nif].tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) == 0)
goto dev_ioctl_err;
m = dpc->wmbufs[eidx].m_table[len_of_mbuf - 1];
iph = rte_pktmbuf_mtod_offset(m, struct iphdr *, sizeof(struct ether_hdr));
tcph = (struct tcphdr *)((uint8_t *)iph + (iph->ihl<<2));
m->l2_len = sizeof(struct ether_hdr);
m->l3_len = (iph->ihl<<2);
m->l4_len = (tcph->doff<<2);
m->ol_flags = PKT_TX_TCP_CKSUM | PKT_TX_IP_CKSUM | PKT_TX_IPV4;
tcph->check = rte_ipv4_phdr_cksum((struct ipv4_hdr *)iph, m->ol_flags);
break;
case PKT_RX_IP_CSUM:
if ((dev_info[nif].rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) == 0)
goto dev_ioctl_err;
break;
case PKT_RX_TCP_CSUM:
if ((dev_info[nif].rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM) == 0)
goto dev_ioctl_err;
break;
case PKT_TX_TCPIP_CSUM_PEEK:
if ((dev_info[nif].tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) == 0)
goto dev_ioctl_err;
if ((dev_info[nif].tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) == 0)
goto dev_ioctl_err;
break;
default:
goto dev_ioctl_err;
}
return 0;
dev_ioctl_err:
return -1;
}
/*----------------------------------------------------------------------------*/
io_module_func onvm_module_func = {
.load_module = onvm_load_module,
.init_handle = onvm_init_handle,
.link_devices = onvm_link_devices,
.release_pkt = onvm_release_pkt,
.send_pkts = onvm_send_pkts,
.get_wptr = onvm_get_wptr,
.recv_pkts = onvm_recv_pkts,
.get_rptr = onvm_get_rptr,
.select = onvm_select,
.destroy_handle = onvm_destroy_handle,
.dev_ioctl = onvm_dev_ioctl
};
/*----------------------------------------------------------------------------*/
#else
io_module_func onvm_module_func = {
.load_module = NULL,
.init_handle = NULL,
.link_devices = NULL,
.release_pkt = NULL,
.send_pkts = NULL,
.get_wptr = NULL,
.recv_pkts = NULL,
.get_rptr = NULL,
.select = NULL,
.destroy_handle = NULL,
.dev_ioctl = NULL
};
/*----------------------------------------------------------------------------*/
#endif /* ENABLE_ONVM */

View File

@ -86,7 +86,7 @@ RBManagerCreate(mtcp_manager_t mtcp, size_t chunk_size, uint32_t cnum)
rbm->chunk_size = chunk_size;
rbm->cnum = cnum;
#ifndef DISABLE_DPDK
#if ! defined(DISABLE_DPDK) && ! defined(ENABLE_ONVM)
char pool_name[RTE_MEMPOOL_NAMESIZE];
sprintf(pool_name, "rbm_pool_%u", mtcp->ctx->cpu);
rbm->mp = (mem_pool_t)MPCreate(pool_name, chunk_size, (uint64_t)chunk_size * cnum);
@ -98,7 +98,7 @@ RBManagerCreate(mtcp_manager_t mtcp, size_t chunk_size, uint32_t cnum)
free(rbm);
return NULL;
}
#ifndef DISABLE_DPDK
#if ! defined(DISABLE_DPDK) && ! defined(ENABLE_ONVM)
sprintf(pool_name, "frag_mp_%u", mtcp->ctx->cpu);
rbm->frag_mp = (mem_pool_t)MPCreate(pool_name, sizeof(struct fragment_ctx),
sizeof(struct fragment_ctx) * cnum);

View File

@ -36,7 +36,7 @@ SBManagerCreate(mtcp_manager_t mtcp, size_t chunk_size, uint32_t cnum)
sbm->chunk_size = chunk_size;
sbm->cnum = cnum;
#ifndef DISABLE_DPDK
#if !defined(DISABLE_DPDK) && !defined(ENABLE_ONVM)
char pool_name[RTE_MEMPOOL_NAMESIZE];
sprintf(pool_name, "sbm_pool_%d", mtcp->ctx->cpu);
sbm->mp = (mem_pool_t)MPCreate(pool_name, chunk_size, (uint64_t)chunk_size * cnum);