From 77ba42d8b77609c9852b89496f232c92dc3cbab6 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 4 Aug 2020 23:42:07 +0300 Subject: [PATCH] Remove dpdk-iface module --- .buildtest_dpdk.sh | 3 - dpdk-iface-kmod/Makefile | 43 ---- dpdk-iface-kmod/dpdk_iface.c | 288 ---------------------- dpdk-iface-kmod/dpdk_iface.h | 183 -------------- dpdk-iface-kmod/dpdk_iface_common.h | 46 ---- dpdk-iface-kmod/dpdk_iface_main.c | 345 --------------------------- mtcp/src/dpdk_module.c | 80 ------- mtcp/src/include/dpdk_iface_common.h | 1 - mtcp/src/io_module.c | 1 - mtcp/src/onvm_module.c | 75 ------ setup_mtcp_dpdk_env.sh | 21 -- setup_mtcp_onvm_env.sh | 21 -- 12 files changed, 1107 deletions(-) delete mode 100644 dpdk-iface-kmod/Makefile delete mode 100644 dpdk-iface-kmod/dpdk_iface.c delete mode 100644 dpdk-iface-kmod/dpdk_iface.h delete mode 100644 dpdk-iface-kmod/dpdk_iface_common.h delete mode 100644 dpdk-iface-kmod/dpdk_iface_main.c delete mode 120000 mtcp/src/include/dpdk_iface_common.h diff --git a/.buildtest_dpdk.sh b/.buildtest_dpdk.sh index 9520016..faba955 100644 --- a/.buildtest_dpdk.sh +++ b/.buildtest_dpdk.sh @@ -8,9 +8,6 @@ sed -i -e 's/O_TO_EXE_STR =/\$(shell if [ \! -d \${RTE_SDK}\/\${RTE_TARGET}\/lib cd dpdk/ make install T=x86_64-native-linuxapp-gcc cd .. -cd dpdk-iface-kmod -make -cd .. autoreconf -ivf ./configure --with-dpdk-lib=$RTE_SDK/$RTE_TARGET make diff --git a/dpdk-iface-kmod/Makefile b/dpdk-iface-kmod/Makefile deleted file mode 100644 index 93da303..0000000 --- a/dpdk-iface-kmod/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -#-------------------------------------------------------------------------# -ifeq ($(RTE_SDK),) -$(error "Please define RTE_SDK environment variable") -endif - -ifeq ($(RTE_TARGET),) -$(error "Please define RTE_TARGET environment variable") -endif -#-------------------------------------------------------------------------# -include $(RTE_SDK)/mk/rte.vars.mk -CC=gcc -obj-m=dpdk_iface.o -DPDK_MACHINE_LINKER_FLAGS=$${RTE_SDK}/$${RTE_TARGET}/lib/ldflags.txt -DPDK_MACHINE_LDFLAGS=$(shell cat ${DPDK_MACHINE_LINKER_FLAGS}) -DPDK_INC=${RTE_SDK}/${RTE_TARGET}/include -DPDK_LIB=${RTE_SDK}/${RTE_TARGET}/lib -appname=dpdk_iface_main -#-------------------------------------------------------------------------# -ifeq ($V,) # no echo - export MSG=@echo - export HIDE=@ -else - export MSG=@\# - export HIDE= -endif -#-------------------------------------------------------------------------# -all: dpdk_iface.c $(appname) $(appname).c - make -C /lib/modules/$(shell uname -r)/build/ M=$(PWD) modules - -$(appname): $(appname).c - $(MSG) " CC $<" - $(HIDE) $(CC) -g $(CFLAGS) $(appname).c -g -o $(appname) -g \ - -I${DPDK_INC} -include $(DPDK_INC)/rte_config.h \ - -L$(DPDK_LIB) ${DPDK_MACHINE_LDFLAGS} -lpthread - -clean: - make -C /lib/modules/$(shell uname -r)/build/ M=$(PWD) clean - $(MSG) " CLEAN $(appname)" - $(HIDE) rm -rf *~ *.o *.ko dpdk_iface_main - -run: all - sudo ./$(appname) -#-------------------------------------------------------------------------# diff --git a/dpdk-iface-kmod/dpdk_iface.c b/dpdk-iface-kmod/dpdk_iface.c deleted file mode 100644 index 18b19e9..0000000 --- a/dpdk-iface-kmod/dpdk_iface.c +++ /dev/null @@ -1,288 +0,0 @@ -/*- - * GPL LICENSE SUMMARY - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * The full GNU General Public License is included in this distribution - * in the file called LICENSE.GPL. - * - * Contact Information: - * Intel Corporation - */ -/*--------------------------------------------------------------------------*/ -#include -#include -#include -#include -#include -#include -#include -#include "dpdk_iface.h" -/*--------------------------------------------------------------------------*/ -struct stats_struct sarrays[MAX_DEVICES][MAX_QID] = {{{0, 0, 0, 0, 0, 0, 0, 0, 0}}}; -struct stats_struct old_sarrays[MAX_DEVICES][MAX_QID] = {{{0, 0, 0, 0, 0, 0, 0, 0, 0}}}; -static int major_no = -1; -/*--------------------------------------------------------------------------*/ -static int -update_stats(struct stats_struct *stats) -{ - uint8_t qid = stats->qid; - uint8_t device = stats->dev; - struct stats_struct *old_sarray = &old_sarrays[device][qid]; - struct stats_struct *sarray = &sarrays[device][qid]; - - if (unlikely(sarrays[device][qid].rx_bytes > stats->rx_bytes || - sarrays[device][qid].tx_bytes > stats->tx_bytes)) { - /* mTCP app restarted?? */ - old_sarray->rx_bytes += sarray->rx_bytes; - old_sarray->rx_pkts += sarray->rx_pkts; - old_sarray->tx_bytes += sarray->tx_bytes; - old_sarray->tx_pkts += sarray->tx_pkts; - old_sarray->rmiss += sarray->rmiss; - old_sarray->rerr += sarray->rerr; - old_sarray->terr += sarray->terr; - } - - sarray->rx_bytes = stats->rx_bytes; - sarray->rx_pkts = stats->rx_pkts; - sarray->tx_bytes = stats->tx_bytes; - sarray->tx_pkts = stats->tx_pkts; - sarray->rmiss = stats->rmiss; - sarray->rerr = stats->rerr; - sarray->terr = stats->terr; - -#if 0 - printk(KERN_ALERT "%s: Dev: %d, Qid: %d, RXP: %llu, " - "RXB: %llu, TXP: %llu, TXB: %llu\n", - device, qid, - THIS_MODULE->name, - (long long unsigned int)sarray->rx_pkts, - (long long unsigned int)sarray->rx_bytes, - (long long unsigned int)sarray->tx_pkts, - (long long unsigned int)sarray->tx_bytes); -#endif - return 0; -} -/*--------------------------------------------------------------------------*/ -static void -clear_all_netdevices(void) -{ - struct net_device *netdev, *dpdk_netdev; - uint8_t freed; - - do { - dpdk_netdev = NULL; - freed = 0; - write_lock(&dev_base_lock); - netdev = first_net_device(&init_net); - while (netdev) { - if (strncmp(netdev->name, IFACE_PREFIX, - strlen(IFACE_PREFIX)) == 0) { - dpdk_netdev = netdev; - break; - } - netdev = next_net_device(netdev); - } - write_unlock(&dev_base_lock); - if (dpdk_netdev) { - unregister_netdev(dpdk_netdev); - free_netdev(dpdk_netdev); - freed = 1; - } - } while (freed); -} -/*--------------------------------------------------------------------------*/ -int -igb_net_open(struct inode *inode, struct file *filp) -{ - return 0; -} -/*--------------------------------------------------------------------------*/ -int -igb_net_release(struct inode *inode, struct file *filp) -{ - return 0; -} -/*--------------------------------------------------------------------------*/ -long -igb_net_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) -{ - int ret = 0; - unsigned char mac_addr[ETH_ALEN]; - struct net_device *netdev; - struct stats_struct ss; - struct net_adapter *adapter = NULL; - struct PciDevice pd; - - switch (cmd) { - case SEND_STATS: - ret = copy_from_user(&ss, - (struct stats_struct __user *)arg, - sizeof(struct stats_struct)); - if (ret) - return -EFAULT; - ret = update_stats(&ss); - break; - case CREATE_IFACE: - ret = copy_from_user(&pd, - (PciDevice __user *)arg, - sizeof(PciDevice)); - ret = copy_from_user(mac_addr, - (unsigned char __user *)pd.ports_eth_addr, - ETH_ALEN); - if (!ret) { - /* first check whether the entry does not exist */ - read_lock(&dev_base_lock); - netdev = first_net_device(&init_net); - while (netdev) { - if (memcmp(netdev->dev_addr, mac_addr, ETH_ALEN) == 0) { - read_unlock(&dev_base_lock); - printk(KERN_ERR "%s: port already registered!\n", THIS_MODULE->name); - return -EINVAL; - } - netdev = next_net_device(netdev); - } - read_unlock(&dev_base_lock); - - /* initialize the corresponding netdev */ - netdev = alloc_etherdev(sizeof(struct net_adapter)); - if (!netdev) { - ret = -ENOMEM; - } else { - SET_NETDEV_DEV(netdev, NULL); - adapter = netdev_priv(netdev); - adapter->netdev = netdev; - netdev_assign_netdev_ops(netdev); - memcpy(netdev->dev_addr, mac_addr, ETH_ALEN); - strcpy(netdev->name, IFACE_PREFIX"%d"); - ret = register_netdev(netdev); - if (ret) - goto fail_ioremap; - adapter->netdev_registered = true; - - if ((ret=sscanf(netdev->name, IFACE_PREFIX"%hu", &adapter->bd_number)) <= 0) - goto fail_bdnumber; - - printk(KERN_INFO "%s: ifindex picked: %hu\n", - THIS_MODULE->name, adapter->bd_number); - /* reset nstats */ - memset(&adapter->nstats, 0, sizeof(struct net_device_stats)); - /* set 'fake' pci address */ - memcpy(&adapter->pa, &pd.pa, sizeof(struct PciAddress)); - ret = copy_to_user((unsigned char __user *)arg, - netdev->name, - IFNAMSIZ); - if (ret) { - printk(KERN_INFO "%s: Interface %s copy to user failed!\n", - THIS_MODULE->name, netdev->name); - ret = -1; - goto fail_pciaddr; - } - /* set numa locality */ - adapter->numa_socket = pd.numa_socket; - } - } - break; - case CLEAR_IFACE: - clear_all_netdevices(); - break; - - case FETCH_PCI_ADDRESS: - ret = copy_from_user(&pd, - (PciDevice __user *)arg, - sizeof(PciDevice)); - if (!ret) { - read_lock(&dev_base_lock); - netdev = first_net_device(&init_net); - while (netdev) { - if (strcmp(netdev->name, pd.ifname) == 0) { - read_unlock(&dev_base_lock); - printk(KERN_INFO "%s: Passing PCI info of %s to user\n", - THIS_MODULE->name, pd.ifname); - adapter = netdev_priv(netdev); - ret = copy_to_user(&((PciDevice __user *)arg)->pa, - &adapter->pa, - sizeof(struct PciAddress)); - if (ret) return -1; - ret = copy_to_user(&((PciDevice __user *)arg)->numa_socket, - &adapter->numa_socket, - sizeof(adapter->numa_socket)); - if (ret) return -1; - return 0; - } - netdev = next_net_device(netdev); - } - read_unlock(&dev_base_lock); - ret = -1; - } - break; - default: - ret = -ENOTTY; - break; - } - - - return ret; - fail_pciaddr: - fail_bdnumber: - unregister_netdev(netdev); - fail_ioremap: - free_netdev(netdev); - return ret; -} -/*--------------------------------------------------------------------------*/ -static struct file_operations igb_net_fops = { - .open = igb_net_open, - .release = igb_net_release, - .unlocked_ioctl = igb_net_ioctl, -}; -/*--------------------------------------------------------------------------*/ -static int __init -iface_pci_init_module(void) -{ - int ret; - - ret = register_chrdev(0 /* MAJOR */, - DEV_NAME /*NAME*/, - &igb_net_fops); - if (ret < 0) { - printk(KERN_ERR "%s: register_chrdev failed\n", - THIS_MODULE->name); - return ret; - } - - printk(KERN_INFO "%s: Loaded\n", - THIS_MODULE->name); - - /* record major number */ - major_no = ret; - - return 0; -} -/*--------------------------------------------------------------------------*/ -static void __exit -iface_pci_exit_module(void) -{ - clear_all_netdevices(); - unregister_chrdev(major_no, DEV_NAME); -} -/*--------------------------------------------------------------------------*/ -module_init(iface_pci_init_module); -module_exit(iface_pci_exit_module); - -MODULE_DESCRIPTION("Interface driver for DPDK devices"); -MODULE_LICENSE("BSD"); -MODULE_AUTHOR("mtcp@list.ndsl.kaist.edu"); -/*--------------------------------------------------------------------------*/ diff --git a/dpdk-iface-kmod/dpdk_iface.h b/dpdk-iface-kmod/dpdk_iface.h deleted file mode 100644 index a9264fa..0000000 --- a/dpdk-iface-kmod/dpdk_iface.h +++ /dev/null @@ -1,183 +0,0 @@ -#ifndef __DPDK_IFACE_H__ -#define __DPDK_IFACE_H__ -/*--------------------------------------------------------------------------*/ -#include -#include "dpdk_iface_common.h" -/*--------------------------------------------------------------------------*/ -#define IFACE_PREFIX "dpdk" -/*--------------------------------------------------------------------------*/ -/** - * net adapter private struct - */ -struct net_adapter { - struct net_device *netdev; - unsigned char mac_addr[ETH_ALEN]; - u16 bd_number; - bool netdev_registered; - int numa_socket; - struct net_device_stats nstats; - struct PciAddress pa; -}; -/*--------------------------------------------------------------------------*/ -/** - * stats struct passed on from user space to the driver - */ -struct stats_struct { - uint64_t tx_bytes; - uint64_t tx_pkts; - uint64_t rx_bytes; - uint64_t rx_pkts; - uint64_t rmiss; - uint64_t rerr; - uint64_t terr; - uint8_t qid; - uint8_t dev; -}; -/*--------------------------------------------------------------------------*/ -/* sarray declaration */ -extern struct stats_struct sarrays[MAX_DEVICES][MAX_QID]; -extern struct stats_struct old_sarrays[MAX_DEVICES][MAX_QID]; -/*----------------------------------------------------------------------------*/ -/** - * dummy function whenever a device is `opened' - */ -static int -netdev_open(struct net_device *netdev) -{ - (void)netdev; - return 0; -} -/*----------------------------------------------------------------------------*/ -/** - * dummy function for retrieving net stats - */ -static struct net_device_stats * -netdev_stats(struct net_device *netdev) -{ - struct net_adapter *adapter; - struct stats_struct *old_sarray = NULL; - struct stats_struct *sarray = NULL; - int i, ifdx; - - adapter = netdev_priv(netdev); - ifdx = adapter->bd_number; - - if (ifdx >= MAX_DEVICES) - printk(KERN_ERR "ifindex value: %d is greater than MAX_DEVICES!\n", - ifdx); - - adapter->nstats.rx_packets = adapter->nstats.tx_packets = 0; - adapter->nstats.rx_bytes = adapter->nstats.tx_bytes = 0; - - for (i = 0; i < MAX_QID; i++) { - sarray = &sarrays[ifdx][i]; - old_sarray = &old_sarrays[ifdx][i]; - - adapter->nstats.rx_packets += sarray->rx_pkts + old_sarray->rx_pkts; - adapter->nstats.rx_bytes += sarray->rx_bytes + old_sarray->rx_bytes; - adapter->nstats.tx_packets += sarray->tx_pkts + old_sarray->tx_pkts; - adapter->nstats.tx_bytes += sarray->tx_bytes + old_sarray->tx_bytes; - adapter->nstats.rx_missed_errors += sarray->rmiss + old_sarray->rmiss; - adapter->nstats.rx_frame_errors += sarray->rerr + old_sarray->rerr; - adapter->nstats.tx_errors += sarray->terr + old_sarray->terr; - } - -#if 0 - printk(KERN_INFO "ifdx: %d, rxp: %llu, rxb: %llu, txp: %llu, txb: %llu\n", - ifdx, - (long long unsigned int)adapter->nstats.rx_packets, - (long long unsigned int)adapter->nstats.rx_bytes, - (long long unsigned int)adapter->nstats.tx_packets, - (long long unsigned int)adapter->nstats.tx_bytes); -#endif - return &adapter->nstats; -} -/*----------------------------------------------------------------------------*/ -/** - * dummy function for setting features - */ -static int -netdev_set_features(struct net_device *netdev, netdev_features_t features) -{ - (void)netdev; - (void)features; - return 0; -} -/*----------------------------------------------------------------------------*/ -/** - * dummy function for fixing features - */ -static netdev_features_t -netdev_fix_features(struct net_device *netdev, netdev_features_t features) -{ - (void)netdev; - (void)features; - return 0; -} -/*----------------------------------------------------------------------------*/ -/** - * dummy function that returns void - */ -static void -netdev_no_ret(struct net_device *netdev) -{ - (void)netdev; - return; -} -/*----------------------------------------------------------------------------*/ -/** - * dummy tx function - */ -static int -netdev_xmit(struct sk_buff *skb, struct net_device *netdev) { - (void)netdev; - (void)skb; - return 0; -} -/*----------------------------------------------------------------------------*/ -/** - * A naive net_device_ops struct to get the interface visible to the OS - */ -static const struct net_device_ops netdev_ops = { - .ndo_open = netdev_open, - .ndo_stop = netdev_open, - .ndo_start_xmit = netdev_xmit, - .ndo_set_rx_mode = (void*)netdev_no_ret, - .ndo_validate_addr = netdev_open, - .ndo_set_mac_address = NULL, - .ndo_change_mtu = NULL, - .ndo_tx_timeout = (void*)netdev_no_ret, - .ndo_vlan_rx_add_vid = NULL, - .ndo_vlan_rx_kill_vid = NULL, - .ndo_do_ioctl = NULL, - .ndo_set_vf_mac = NULL, - .ndo_set_vf_vlan = NULL, -#if LINUX_VERSION_CODE <= KERNEL_VERSION(3, 15, 0) - .ndo_set_vf_tx_rate = NULL, -#else - .ndo_set_vf_rate = NULL, -#endif - .ndo_set_vf_spoofchk = NULL, - .ndo_get_vf_config = NULL, - .ndo_get_stats = netdev_stats, - .ndo_setup_tc = NULL, -#ifdef CONFIG_NET_POLL_CONTROLLER - .ndo_poll_controller = (void*)netdev_no_ret, - .ndo_netpoll_setup = NULL, - .ndo_netpoll_cleanup = NULL, -#endif - .ndo_set_features = netdev_set_features, - .ndo_fix_features = netdev_fix_features, - .ndo_fdb_add = NULL, -}; -/*----------------------------------------------------------------------------*/ -/** - * assignment function - */ -void -netdev_assign_netdev_ops(struct net_device *dev) -{ - dev->netdev_ops = &netdev_ops; -} -/*----------------------------------------------------------------------------*/ -#endif /* __DPDK_IFACE_H__ */ diff --git a/dpdk-iface-kmod/dpdk_iface_common.h b/dpdk-iface-kmod/dpdk_iface_common.h deleted file mode 100644 index 416a2f5..0000000 --- a/dpdk-iface-kmod/dpdk_iface_common.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef __DPDK_IFACE_COMMON_H__ -#define __DPDK_IFACE_COMMON_H__ -/*--------------------------------------------------------------------------*/ -/* for ETH_ALEN */ -#ifndef __KERNEL__ -#include -#endif -#include -/*--------------------------------------------------------------------------*/ -/* dev name */ -#define DEV_NAME "dpdk-iface" -#define DEV_PATH "/dev/"DEV_NAME -#define DEV_PROC_PATH "/proc/devices" -/* ioctl# */ -#define SEND_STATS 0 -#define CREATE_IFACE 1 -#define CLEAR_IFACE 4 -#define FETCH_PCI_ADDRESS 5 -/* max qid */ -#define MAX_QID 128 -#ifndef MAX_DEVICES -#define MAX_DEVICES 128 -#endif -#define PCI_DOM "%04hX" -#define PCI_BUS "%02hhX" -#define PCI_DEVICE "%02hhX" -#define PCI_FUNC "%01hhX" -#define PCI_LENGTH 13 -/*--------------------------------------------------------------------------*/ -typedef struct PciAddress { - uint16_t domain; - uint8_t bus; - uint8_t device; - uint8_t function; -} PciAddress; -/*--------------------------------------------------------------------------*/ -typedef struct PciDevice { - union { - uint8_t *ports_eth_addr; - char ifname[IFNAMSIZ]; - }; - PciAddress pa; - int numa_socket; -} PciDevice __attribute__((aligned(__BITS_PER_LONG))); -/*--------------------------------------------------------------------------*/ -#endif /* __DPDK_IFACE_COMMON_H__ */ diff --git a/dpdk-iface-kmod/dpdk_iface_main.c b/dpdk-iface-kmod/dpdk_iface_main.c deleted file mode 100644 index 143cea4..0000000 --- a/dpdk-iface-kmod/dpdk_iface_main.c +++ /dev/null @@ -1,345 +0,0 @@ -#define _GNU_SOURCE 1 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "dpdk_iface_common.h" -/*--------------------------------------------------------------------------*/ -//#define DEBUG 1 -#define SYSFS_PCI_DRIVER_PATH "/sys/bus/pci/drivers/" -#define SYSFS_PCI_IGB_UIO SYSFS_PCI_DRIVER_PATH"igb_uio" -#define SYSFS_PCI_VFIO_PCI SYSFS_PCI_DRIVER_PATH"vfio-pci" -#define SYSFS_PCI_UIOPCIGEN SYSFS_PCI_DRIVER_PATH"uio_pci_generic" -#define RTE_ARGC_MAX (RTE_MAX_ETHPORTS << 1) + 7 -/*--------------------------------------------------------------------------*/ -typedef struct { - PciDevice pd; - struct rte_eth_dev_info dev_details; - struct ether_addr ports_eth_addr; -} DevInfo; - -static DevInfo di[RTE_MAX_ETHPORTS]; -/*--------------------------------------------------------------------------*/ -/** - * Really crappy version for detecting pci entries.. - * but it should work. - */ -int -IsPciEnt(const struct dirent *entry) -{ - if (entry->d_type == DT_LNK && - strstr(entry->d_name, ":") != NULL) - return 1; - - return 0; -} -/*--------------------------------------------------------------------------*/ -/** - * Similar to strverscmp(), but sorts in hexadecimal context - */ -int -localversionsort(const void *elem1, const void *elem2) -{ - uint16_t domain1, domain2; - uint8_t bus1, bus2, device1, device2, function1, function2; - DevInfo *d1 = (DevInfo *)elem1; - DevInfo *d2 = (DevInfo *)elem2; - - domain1 = d1->pd.pa.domain; - domain2 = d2->pd.pa.domain; - bus1 = d1->pd.pa.bus; - bus2 = d2->pd.pa.bus; - device1 = d1->pd.pa.device; - device2 = d2->pd.pa.device; - function1 = d1->pd.pa.function; - function2 = d2->pd.pa.function; - - if (domain1 < domain2) return -1; - if (domain2 < domain1) return 1; - - if (bus1 < bus2) return -1; - if (bus2 < bus1) return 1; - - if (device1 < device2) return -1; - if (device2 < device1) return 1; - - if (function1 < function2) - return -1; - if (function2 < function1) - return 1; - - return 0; -} -/*--------------------------------------------------------------------------*/ -int -probe_all_rte_devices(char **argv, int *argc) -{ - struct dirent **dirlist; - int pci_index, total_files, i, j; - - /* reset pci_index */ - pci_index = 0; - - for (j = 0; j < 3; j++) { - switch (j) { - case 0: - /* scan igb_uio first */ - total_files = scandir(SYSFS_PCI_IGB_UIO, &dirlist, - IsPciEnt, versionsort); - break; - case 1: - /* scan vfio_pci next */ - total_files = scandir(SYSFS_PCI_VFIO_PCI, &dirlist, - IsPciEnt, versionsort); - break; - case 2: - /* finally scan uio_pci_generic */ - total_files = scandir(SYSFS_PCI_UIOPCIGEN, &dirlist, - IsPciEnt, versionsort); - break; - default: - fprintf(stderr, "Control can never come here!\n"); - goto panic_err; - } - - for (i = 0; i < total_files; i++, pci_index++) { - argv[*argc] = strdup("-w"); - argv[*argc + 1] = strdup(dirlist[i]->d_name); - if (argv[*argc] == NULL || - argv[*argc + 1] == NULL) - goto alloc_err; - *argc += 2; - if (sscanf(dirlist[i]->d_name, PCI_DOM":"PCI_BUS":" - PCI_DEVICE"."PCI_FUNC, - &di[pci_index].pd.pa.domain, - &di[pci_index].pd.pa.bus, - &di[pci_index].pd.pa.device, - &di[pci_index].pd.pa.function) != 4) - goto sscanf_err; - free(dirlist[i]); - } - - //free(dirlist); - } - - /* now sort all recorded entries */ - qsort(di, pci_index, sizeof(DevInfo), localversionsort); - return pci_index; - sscanf_err: - fprintf(stderr, "Unable to retrieve pci address!\n"); - exit(EXIT_FAILURE); - alloc_err: - fprintf(stderr, "Can't allocate memory for argv items!\n"); - exit(EXIT_FAILURE); - panic_err: - fprintf(stderr, "Could not open the directory!\n"); - exit(EXIT_FAILURE); -} -/*--------------------------------------------------------------------------*/ -int -fetch_major_no() -{ - FILE *f; - int major_no; - char *line; - size_t len; - char dummy[512]; - - major_no = -1; - len = 0; - line = NULL; - - f = fopen(DEV_PROC_PATH, "r"); - if (f == NULL) { - fprintf(stderr, "Can't open %s file\n", DEV_PROC_PATH); - return -1; - } - - while (getline(&line, &len, f) != -1) { - if (strstr(line, DEV_NAME) != NULL) { - if (sscanf(line, "%d %s", &major_no, dummy) == 2) { - free(line); - break; - } - } - free(line); - line = NULL; - len = 0; - } - - /* close the file descriptor */ - fclose(f); - - return major_no; -} -/*--------------------------------------------------------------------------*/ -int -main(int argc, char **argv) -{ - int ret, fd, num_devices, i; - dev_t dev; - char *cpumaskbuf = "0x1"; - char *mem_channels = "4"; - char *rte_argv[RTE_ARGC_MAX] = {"", - "-c", - cpumaskbuf, - "-n", - mem_channels, - "--proc-type=auto" - }; - int rte_argc = 6; - - ret = probe_all_rte_devices(rte_argv, &rte_argc); - -#if DEBUG - for (i = 0; i < ret; i++) { - fprintf(stderr, "Pci Address: %04hX:%02hhX:%02hhX.%01hhX\n", - di[i].pd.pa.domain, - di[i].pd.pa.bus, - di[i].pd.pa.device, - di[i].pd.pa.function); - } -#endif - - if (geteuid()) { - fprintf(stderr, "[CAUTION] Run the app as root!\n"); - exit(EXIT_FAILURE); - } - - /* remove previously created dpdk-iface device node file */ - fprintf(stderr, "Removing existing device node entry..."); - ret = remove(DEV_PATH); - fprintf(stderr, (ret == 0) ? "\033[32m done. \033[0m \n" : - "\033[32m not present. \033[0m \n"); - - /* create dpdk-iface device node entry */ -#if 0 - dev = makedev(MAJOR_NO, 0); -#else - dev = makedev(fetch_major_no(), 0); -#endif - ret = mknod(DEV_PATH, S_IFCHR | O_RDWR, dev); - if (ret == 0) - fprintf(stderr, "Creating device node entry..."); - else { - fprintf(stderr, "Failed to create device node entry\n"); - return EXIT_FAILURE; - } - - fprintf(stderr, "\033[32m done. \033[0m \n"); - - /* setting permissions on the device node entry */ - ret = chmod(DEV_PATH, - S_IRGRP | S_IROTH | S_IRUSR | - S_IWGRP | S_IWOTH | S_IWUSR); - - if (ret == 0) - fprintf(stderr, "Setting permissions on the device node entry..."); - else { - fprintf(stderr, "Failed to set permissions on the device node entry\n"); - return EXIT_FAILURE; - } - - fprintf(stderr, "\033[32m done. \033[0m \n"); - -#if RTE_VERSION < RTE_VERSION_NUM(17, 05, 0, 16) - rte_set_log_level(RTE_LOG_EMERG); -#else - rte_log_set_global_level(RTE_LOG_EMERG); -#endif - - fprintf(stderr, "Scanning the system for dpdk-compatible devices..."); - /* initialize the rte env first */ - ret = rte_eal_init(rte_argc, rte_argv); - - /* get total count of detected ethernet ports */ - num_devices = rte_eth_dev_count(); - if (num_devices == 0) { - fprintf(stderr, "No Ethernet port detected!\n"); - exit(EXIT_FAILURE); - } - - for (ret = 0; ret < num_devices; ret++) { - di[ret].pd.ports_eth_addr = &di[ret].ports_eth_addr.addr_bytes[0]; - /* get mac addr entries of detected dpdk ports */ - rte_eth_macaddr_get(ret, &di[ret].ports_eth_addr); - /* check port capabailties/info */ - rte_eth_dev_info_get(ret, &di[ret].dev_details); - /* get numa socket location for future socket-mem field */ - if ((di[ret].pd.numa_socket=rte_eth_dev_socket_id(ret)) == -1) { - fprintf(stderr, "Can't determine socket ID!\n"); - exit(EXIT_FAILURE); - } - } - - fprintf(stderr, "\033[32m done. \033[0m \n"); - - /* open the device node first */ - fd = open(DEV_PATH, O_RDWR); - if (fd == -1) { - fprintf(stderr, "Failed to open %s for port detection!\n", - DEV_PATH); - exit(EXIT_FAILURE); - } - - /* clear all previous entries */ - fprintf(stderr, "Clearing previous entries\n"); - ret = ioctl(fd, CLEAR_IFACE, di[0].ports_eth_addr.addr_bytes); - if (ret == -1) { - fprintf(stderr, "ioctl call failed!\n"); - return EXIT_FAILURE; - } - - /* register the newly detected dpdk ports */ - for (ret = 0; ret < num_devices; ret++) { - if (strcmp(di[ret].dev_details.driver_name, "net_mlx4") && - strcmp(di[ret].dev_details.driver_name, "net_mlx5")) { - fprintf(stderr, "Registering port %d (%02X:%02X:%02X:%02X:%02X:%02X) to mTCP stack", - ret, - di[ret].ports_eth_addr.addr_bytes[0], - di[ret].ports_eth_addr.addr_bytes[1], - di[ret].ports_eth_addr.addr_bytes[2], - di[ret].ports_eth_addr.addr_bytes[3], - di[ret].ports_eth_addr.addr_bytes[4], - di[ret].ports_eth_addr.addr_bytes[5]); - di[ret].pd.ports_eth_addr = di[ret].ports_eth_addr.addr_bytes; - - if (ioctl(fd, CREATE_IFACE, &di[ret].pd) == -1) { - fprintf(stderr, "ioctl call failed!\n"); - } - fprintf(stderr, " (%s).\n", - di[ret].pd.ifname); - } - } - - /* close the fd */ - close(fd); - -#if 0 - /* - * XXX: It seems that there is a bug in the RTE SDK. - * The dynamically allocated rte_argv params are left - * as dangling pointers. Freeing them causes program - * to crash. - */ - - /* free up all resources */ - for (; rte_argc >= 6; rte_argc--) { - if (rte_argv[rte_argc] != NULL) { - fprintf(stderr, "Cleaning up rte_argv[%d]: %s (%p)\n", - rte_argc, rte_argv[rte_argc], rte_argv[rte_argc]); - free(rte_argv[rte_argc]); - rte_argv[rte_argc] = NULL; - } - } -#endif - return EXIT_SUCCESS; -} -/*--------------------------------------------------------------------------*/ diff --git a/mtcp/src/dpdk_module.c b/mtcp/src/dpdk_module.c index b32249e..c688254 100644 --- a/mtcp/src/dpdk_module.c +++ b/mtcp/src/dpdk_module.c @@ -16,15 +16,6 @@ /* for delay funcs */ #include #include -#define ENABLE_STATS_IOCTL 1 -#ifdef ENABLE_STATS_IOCTL -/* for close */ -#include -/* for open */ -#include -/* for ioctl */ -#include -#endif /* !ENABLE_STATS_IOCTL */ /* for ip pseudo-chksum */ #include //#define IP_DEFRAG 1 @@ -32,8 +23,6 @@ /* for ip defragging */ #include #endif -/* for ioctl funcs */ -#include /* for retrieving rte version(s) */ #include /*----------------------------------------------------------------------------*/ @@ -201,29 +190,8 @@ struct dpdk_private_context { #ifdef ENABLELRO struct rte_mbuf *cur_rx_m; #endif -#ifdef ENABLE_STATS_IOCTL - int fd; - uint32_t cur_ts; -#endif /* !ENABLE_STATS_IOCTL */ } __rte_cache_aligned; -#ifdef ENABLE_STATS_IOCTL -/** - * stats struct passed on from user space to the driver - */ -struct stats_struct { - uint64_t tx_bytes; - uint64_t tx_pkts; - uint64_t rx_bytes; - uint64_t rx_pkts; - uint64_t rmiss; - uint64_t rerr; - uint64_t terr; - uint8_t qid; - uint8_t dev; -}; -#endif /* !ENABLE_STATS_IOCTL */ - #ifdef IP_DEFRAG /* Should be power of two. */ #define IP_FRAG_TBL_BUCKET_ENTRIES 16 @@ -286,15 +254,6 @@ dpdk_init_handle(struct mtcp_thread_context *ctxt) exit(EXIT_FAILURE); } #endif /* !IP_DEFRAG */ - -#ifdef ENABLE_STATS_IOCTL - dpc->fd = open(DEV_PATH, O_RDWR); - if (dpc->fd == -1) { - TRACE_ERROR("Can't open " DEV_PATH " for context->cpu: %d! " - "Are you using mlx4/mlx5 driver?\n", - ctxt->cpu); - } -#endif /* !ENABLE_STATS_IOCTL */ } /*----------------------------------------------------------------------------*/ int @@ -332,43 +291,10 @@ dpdk_send_pkts(struct mtcp_thread_context *ctxt, int ifidx) /* if there are packets in the queue... flush them out to the wire */ if (dpc->wmbufs[ifidx].len >/*= MAX_PKT_BURST*/ 0) { struct rte_mbuf **pkts; -#ifdef ENABLE_STATS_IOCTL -#ifdef NETSTAT - struct rte_eth_stats stats; - struct stats_struct ss; -#endif -#endif /* !ENABLE_STATS_IOCTL */ int cnt = dpc->wmbufs[ifidx].len; pkts = dpc->wmbufs[ifidx].m_table; #ifdef NETSTAT mtcp->nstat.tx_packets[ifidx] += cnt; -#ifdef ENABLE_STATS_IOCTL - /* only pass stats after >= 1 sec interval */ - if (abs(mtcp->cur_ts - dpc->cur_ts) >= 1000 && - likely(dpc->fd >= 0)) { - /* rte_get_stats is global func, use only for 1 core */ - if (ctxt->cpu == 0) { - rte_eth_stats_get(portid, &stats); - ss.rmiss = stats.imissed; - ss.rerr = stats.ierrors; - ss.terr = stats.oerrors; - } else - ss.rmiss = ss.rerr = ss.terr = 0; - - ss.tx_pkts = mtcp->nstat.tx_packets[ifidx]; - ss.tx_bytes = mtcp->nstat.tx_bytes[ifidx]; - ss.rx_pkts = mtcp->nstat.rx_packets[ifidx]; - ss.rx_bytes = mtcp->nstat.rx_bytes[ifidx]; - ss.qid = ctxt->cpu; - ss.dev = portid; - /* pass the info now */ - if (ioctl(dpc->fd, SEND_STATS, &ss) == -1) - TRACE_ERROR("Can't update iface stats!\n"); - dpc->cur_ts = mtcp->cur_ts; - if (ctxt->cpu == 0) - rte_eth_stats_reset(portid); - } -#endif /* !ENABLE_STATS_IOCTL */ #endif do { /* tx cnt # of packets */ @@ -574,12 +500,6 @@ dpdk_destroy_handle(struct mtcp_thread_context *ctxt) for (i = 0; i < num_devices_attached; i++) free_pkts(dpc->wmbufs[i].m_table, MAX_PKT_BURST); -#ifdef ENABLE_STATS_IOCTL - /* free fd */ - if (dpc->fd >= 0) - close(dpc->fd); -#endif /* !ENABLE_STATS_IOCTL */ - /* free it all up */ free(dpc); } diff --git a/mtcp/src/include/dpdk_iface_common.h b/mtcp/src/include/dpdk_iface_common.h deleted file mode 120000 index b95ec5a..0000000 --- a/mtcp/src/include/dpdk_iface_common.h +++ /dev/null @@ -1 +0,0 @@ -../../../dpdk-iface-kmod/dpdk_iface_common.h \ No newline at end of file diff --git a/mtcp/src/io_module.c b/mtcp/src/io_module.c index 4f11ba0..b86e4a4 100644 --- a/mtcp/src/io_module.c +++ b/mtcp/src/io_module.c @@ -16,7 +16,6 @@ #define RTE_ARGC_MAX (RTE_MAX_ETHPORTS << 1) + 9 /* for dpdk ethernet functions (get mac addresses) */ #include -#include /* for ceil func */ #include /* for retrieving rte version(s) */ diff --git a/mtcp/src/onvm_module.c b/mtcp/src/onvm_module.c index 0344b3f..6bde370 100644 --- a/mtcp/src/onvm_module.c +++ b/mtcp/src/onvm_module.c @@ -16,15 +16,6 @@ /* for delay funcs */ #include #include -#define ENABLE_STATS_IOCTL 1 -#ifdef ENABLE_STATS_IOCTL -/* for close */ -#include -/* for open */ -#include -/* for ioctl */ -#include -#endif /* !ENABLE_STATS_IOCTL */ /* for ip pseudo-chksum */ #include @@ -84,10 +75,6 @@ struct dpdk_private_context { #ifdef ENABLELRO struct rte_mbuf *cur_rx_m; #endif -#ifdef ENABLE_STATS_IOCTL - int fd; - uint32_t cur_ts; -#endif /* !ENABLE_STATS_IOCTL */ } __rte_cache_aligned; /* onvm structs */ @@ -96,23 +83,6 @@ struct rte_ring *rx_ring; struct rte_ring *tx_ring; volatile struct onvm_nf *nf; -#ifdef ENABLE_STATS_IOCTL -/** - * stats struct passed on from user space to the driver - */ -struct stats_struct { - uint64_t tx_bytes; - uint64_t tx_pkts; - uint64_t rx_bytes; - uint64_t rx_pkts; - uint64_t rmiss; - uint64_t rerr; - uint64_t terr; - uint8_t qid; - uint8_t dev; - -}; -#endif /* !ENABLE_STATS_IOCTL */ /*----------------------------------------------------------------------------*/ void onvm_init_handle(struct mtcp_thread_context *ctxt) @@ -155,15 +125,6 @@ onvm_init_handle(struct mtcp_thread_context *ctxt) /* set mbufs queue length to 0 to begin with */ dpc->wmbufs[j].len = 0; } - -#ifdef ENABLE_STATS_IOCTL - dpc->fd = open("/dev/dpdk-iface", O_RDWR); - if (dpc->fd == -1) { - TRACE_ERROR("Can't open /dev/dpdk-iface for context->cpu: %d! " - "Are you using mlx4/mlx5 driver?\n", - ctxt->cpu); - } -#endif /* !ENABLE_STATS_IOCTL */ } /*----------------------------------------------------------------------------*/ int @@ -201,40 +162,10 @@ onvm_send_pkts(struct mtcp_thread_context *ctxt, int nif) /* if there are packets in the queue... flush them out to the wire */ if (dpc->wmbufs[nif].len >/*= MAX_PKT_BURST*/ 0) { struct rte_mbuf **pkts; -#ifdef ENABLE_STATS_IOCTL - struct rte_eth_stats stats; - struct stats_struct ss; -#endif /* !ENABLE_STATS_IOCTL */ int cnt = dpc->wmbufs[nif].len; pkts = dpc->wmbufs[nif].m_table; #ifdef NETSTAT mtcp->nstat.tx_packets[nif] += cnt; -#ifdef ENABLE_STATS_IOCTL - /* only pass stats after >= 1 sec interval */ - if (abs(mtcp->cur_ts - dpc->cur_ts) >= 1000 && - likely(dpc->fd >= 0)) { - /* rte_get_stats is global func, use only for 1 core */ - if (ctxt->cpu == 0) { - rte_eth_stats_get(CONFIG.eths[ifidx].ifindex, &stats); - ss.rmiss = stats.imissed; - ss.rerr = stats.ierrors; - ss.terr = stats.oerrors; - } else - ss.rmiss = ss.rerr = ss.terr = 0; - - ss.tx_pkts = mtcp->nstat.tx_packets[ifidx]; - ss.tx_bytes = mtcp->nstat.tx_bytes[ifidx]; - ss.rx_pkts = mtcp->nstat.rx_packets[ifidx]; - ss.rx_bytes = mtcp->nstat.rx_bytes[ifidx]; - ss.qid = ctxt->cpu; - ss.dev = CONFIG.eths[ifidx].ifindex; - /* pass the info now */ - ioctl(dpc->fd, 0, &ss); - dpc->cur_ts = mtcp->cur_ts; - if (ctxt->cpu == 0) - rte_eth_stats_reset(CONFIG.eths[ifidx].ifindex); - } -#endif /* !ENABLE_STATS_IOCTL */ #endif for (i = 0; i < cnt; i++) { @@ -407,12 +338,6 @@ onvm_destroy_handle(struct mtcp_thread_context *ctxt) for (i = 0; i < num_devices_attached; i++) free_pkts(dpc->wmbufs[i].m_table, MAX_PKT_BURST); -#ifdef ENABLE_STATS_IOCTL - /* free fd */ - if (dpc->fd >= 0) - close(dpc->fd); -#endif /* !ENABLE_STATS_IOCTL */ - /* free it all up */ free(dpc); } diff --git a/setup_mtcp_dpdk_env.sh b/setup_mtcp_dpdk_env.sh index 509a5e0..c79bf98 100755 --- a/setup_mtcp_dpdk_env.sh +++ b/setup_mtcp_dpdk_env.sh @@ -49,24 +49,3 @@ done cd - printf "Set ${GREEN}RTE_SDK$NC env variable as $RTE_SDK\n" printf "Set ${GREEN}RTE_TARGET$NC env variable as $RTE_TARGET\n" - -# Check if you are using an Intel NIC -while true; do - read -p "Are you using an Intel NIC (y/n)? " response - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - esac -done - -# Create interfaces -printf "Creating ${GREEN}dpdk$NC interface entries\n" -cd dpdk-iface-kmod -make -if lsmod | grep dpdk_iface &> /dev/null ; then - : -else - sudo insmod ./dpdk_iface.ko -fi -sudo -E make run -cd .. diff --git a/setup_mtcp_onvm_env.sh b/setup_mtcp_onvm_env.sh index 248e67e..1dcfb18 100755 --- a/setup_mtcp_onvm_env.sh +++ b/setup_mtcp_onvm_env.sh @@ -25,24 +25,3 @@ fi printf "${GREEN}RTE_SDK$NC env variable is set to $RTE_SDK\n" printf "${GREEN}RTE_TARGET$NC env variable is set to $RTE_TARGET\n" - -# Check if you are using an Intel NIC -while true; do - read -p "Are you using an Intel NIC (y/n)? " response - case $response in - [Yy]* ) break;; - [Nn]* ) exit;; - esac -done - -# Create interfaces -printf "Creating ${GREEN}dpdk$NC interface entries\n" -cd dpdk-iface-kmod -make -if lsmod | grep dpdk_iface &> /dev/null ; then - : -else - sudo insmod ./dpdk_iface.ko -fi -sudo -E make run -cd ..